From 1331fe2b7c104e83171081fef45dcd307f15e7a4 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 13:40:03 -0700 Subject: [PATCH 01/95] feat(messaging): add durable messaging contracts Adds the Microsoft.Orleans.Messaging package with the public envelope, hierarchical key, inbox, outbox, handler, typed message, delivery result, and options contracts, plus contract tests. Layer 1 of 4 for durable messaging (#11282 -> #11284 -> #11285 -> #10693). --- Orleans.slnx | 2 + .../Configuration/InboxOptions.cs | 220 ++ src/Orleans.Messaging/DeliveryResult.cs | 55 + src/Orleans.Messaging/DeliveryStatus.cs | 32 + src/Orleans.Messaging/Envelope.cs | 31 + src/Orleans.Messaging/EnvelopeEquivalence.cs | 17 + src/Orleans.Messaging/EnvelopeValidation.cs | 49 + src/Orleans.Messaging/HierarchicalKey.cs | 316 +++ src/Orleans.Messaging/HierarchicalKeyCodec.cs | 48 + .../Hosting/MessageTypeExtensions.cs | 38 + src/Orleans.Messaging/IInbox.cs | 23 + src/Orleans.Messaging/IInboxExtension.cs | 33 + src/Orleans.Messaging/IInboxHandler.cs | 23 + src/Orleans.Messaging/IInboxHandlerContext.cs | 52 + src/Orleans.Messaging/IOutbox.cs | 36 + src/Orleans.Messaging/InboxDispatcher.cs | 37 + src/Orleans.Messaging/InboxExtensions.cs | 127 + src/Orleans.Messaging/InboxMessage.cs | 10 + src/Orleans.Messaging/MessageType.cs | 93 + .../Orleans.Messaging.csproj | 31 + src/Orleans.Messaging/OrleansContracts.txt | 14 + src/Orleans.Messaging/OutboxExtensions.cs | 102 + src/Orleans.Messaging/OutboxMessage.cs | 20 + src/Orleans.Messaging/README.md | 77 + .../DeliveryAndOptionsContractTests.cs | 103 + .../Contracts/EnvelopeContractTests.cs | 315 +++ .../Contracts/HandlerRoutingContractTests.cs | 115 + .../Contracts/HierarchicalKeyTests.cs | 350 +++ .../Contracts/TypedMessageHelperTests.cs | 2120 +++++++++++++++++ .../Contracts/TypedOutboxExtensionsTests.cs | 600 +++++ .../Orleans.Messaging.Tests.csproj | 19 + .../Support/HierarchicalKeyTestExtensions.cs | 15 + .../Support/TestInbox.cs | 24 + 33 files changed, 5147 insertions(+) create mode 100644 src/Orleans.Messaging/Configuration/InboxOptions.cs create mode 100644 src/Orleans.Messaging/DeliveryResult.cs create mode 100644 src/Orleans.Messaging/DeliveryStatus.cs create mode 100644 src/Orleans.Messaging/Envelope.cs create mode 100644 src/Orleans.Messaging/EnvelopeEquivalence.cs create mode 100644 src/Orleans.Messaging/EnvelopeValidation.cs create mode 100644 src/Orleans.Messaging/HierarchicalKey.cs create mode 100644 src/Orleans.Messaging/HierarchicalKeyCodec.cs create mode 100644 src/Orleans.Messaging/Hosting/MessageTypeExtensions.cs create mode 100644 src/Orleans.Messaging/IInbox.cs create mode 100644 src/Orleans.Messaging/IInboxExtension.cs create mode 100644 src/Orleans.Messaging/IInboxHandler.cs create mode 100644 src/Orleans.Messaging/IInboxHandlerContext.cs create mode 100644 src/Orleans.Messaging/IOutbox.cs create mode 100644 src/Orleans.Messaging/InboxDispatcher.cs create mode 100644 src/Orleans.Messaging/InboxExtensions.cs create mode 100644 src/Orleans.Messaging/InboxMessage.cs create mode 100644 src/Orleans.Messaging/MessageType.cs create mode 100644 src/Orleans.Messaging/Orleans.Messaging.csproj create mode 100644 src/Orleans.Messaging/OrleansContracts.txt create mode 100644 src/Orleans.Messaging/OutboxExtensions.cs create mode 100644 src/Orleans.Messaging/OutboxMessage.cs create mode 100644 src/Orleans.Messaging/README.md create mode 100644 test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs create mode 100644 test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj create mode 100644 test/Orleans.Messaging.Tests/Support/HierarchicalKeyTestExtensions.cs create mode 100644 test/Orleans.Messaging.Tests/Support/TestInbox.cs diff --git a/Orleans.slnx b/Orleans.slnx index e4047b5f194..877afac409d 100644 --- a/Orleans.slnx +++ b/Orleans.slnx @@ -42,6 +42,7 @@ + @@ -152,6 +153,7 @@ + diff --git a/src/Orleans.Messaging/Configuration/InboxOptions.cs b/src/Orleans.Messaging/Configuration/InboxOptions.cs new file mode 100644 index 00000000000..f0a6bfaf4ca --- /dev/null +++ b/src/Orleans.Messaging/Configuration/InboxOptions.cs @@ -0,0 +1,220 @@ +using System; + +namespace Orleans.Messaging.Configuration; + +/// +/// Configuration options for the inbox messaging system. +/// +/// +/// +/// These options control the behavior of the inbox, including capacity limits, +/// deduplication tracking, retry behavior, and pump batch sizes. +/// +/// +/// Transport is at-least-once. Deduplication provides effectively-once handler effects only +/// while the processed-message record is retained. Configuration values affect memory usage, +/// throughput, and recovery characteristics. +/// +/// +public class InboxOptions +{ + internal const int MaximumBackoffExponent = 6; + internal const int MaximumBackoffMultiplier = 1 << MaximumBackoffExponent; + + // The runtime multiplies this base delay by at most MaximumBackoffMultiplier. + // Keep the expanded delay within the timer implementation's uint-millisecond limit. + private static readonly TimeSpan MaxSupportedRetryDelay = + TimeSpan.FromTicks( + TimeSpan.FromMilliseconds(uint.MaxValue - 1).Ticks + / MaximumBackoffMultiplier); + + /// + /// Gets or sets the maximum number of pending messages in the inbox. + /// When this limit is reached, new message deliveries will return DeliveryResult.Backpressured(). + /// + /// + /// + /// A lower value (e.g., 100) provides stronger backpressure but may reduce throughput. + /// A higher value (e.g., 10,000) allows more buffering but increases memory usage and recovery time. + /// + /// + /// The inbox capacity is checked before accepting new messages. Messages are persisted to durable + /// storage, so capacity limits affect both in-memory state and storage I/O during recovery. + /// + /// + /// + /// The maximum inbox capacity. Must be greater than zero. Defaults to 1000. + /// + public int MaxCapacity { get; set; } = 1000; + + /// + /// Gets or sets the time window for tracking processed messages to prevent duplicates. + /// Messages that were processed within this window will be rejected with DeliveryResult.Duplicate(). + /// + /// + /// + /// A longer window (e.g., 30 days) provides stronger deduplication guarantees but increases + /// memory usage and storage I/O. A shorter window (e.g., 1 hour) reduces overhead but may + /// allow duplicate processing if retries are delayed. + /// + /// + /// Processed message tracking uses the exact message ID within the receiving inbox, with timestamps. + /// Expired entries are removed atomically when a replay is accepted and are compacted when the + /// inbox writes a snapshot. + /// + /// + /// Consider your retry policies when setting this value. For example, if senders retry for + /// up to 24 hours, set the window to at least 48 hours to ensure deduplication coverage. + /// + /// + /// + /// The deduplication window. Must be greater than zero. Defaults to 7 days. + /// + public TimeSpan DeduplicationWindow { get; set; } = TimeSpan.FromDays(7); + + /// + /// Gets or sets the base delay between retry attempts for outbox delivery and inbox processing. + /// + /// + /// + /// The outbox applies exponential backoff from this duration when delivery fails or the target inbox + /// returns DeliveryResult.Backpressured(). The inbox applies the same backoff after a handler fails. + /// + /// + /// A shorter delay (e.g., 100ms) enables faster recovery when the target processes messages quickly, + /// but may increase CPU usage during sustained backpressure. A longer delay (e.g., 5 seconds) + /// reduces retry overhead but increases latency for message delivery. + /// + /// + /// For high-throughput scenarios where quick recovery from backpressure is important, + /// consider values between 100-500ms. For less time-sensitive workloads, 1-5 seconds is appropriate. + /// + /// + /// + /// The base backpressure retry delay. Must be greater than zero. Defaults to 1 second. + /// + public TimeSpan BackpressureRetryDelay { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets the maximum number of attempts before an inbox message is dead-lettered. + /// + public int MaxProcessingAttempts { get; set; } = 5; + + /// + /// Gets or sets the maximum number of attempts before an outbox message is dead-lettered. + /// + public int MaxDeliveryAttempts { get; set; } = 100; + + /// + /// Gets or sets the maximum age of an outbox message. + /// + public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromDays(1); + + /// + /// Gets or sets how long inbox and outbox dead letters are retained. + /// + public TimeSpan DeadLetterRetentionPeriod { get; set; } = TimeSpan.FromDays(30); + + /// + /// Gets or sets the maximum number of dead letters retained per inbox and per outbox. + /// + public int MaxRetainedDeadLetters { get; set; } = 1000; + + /// + /// Gets or sets the maximum number of inbox messages processed by one messaging pump pass. + /// + public int InboxBatchSize { get; set; } = 32; + + /// + /// Gets or sets the maximum number of outbox messages processed by one messaging pump pass. + /// + public int OutboxBatchSize { get; set; } = 32; + + + /// + /// Validates the configuration values and throws if any are invalid. + /// + /// + /// Thrown if is less than or equal to zero, + /// or if is less than or equal to , + /// or if a retry, retention, or batch option is outside its supported range. + /// + /// + /// This method is typically called by the dependency injection container during service registration + /// to ensure configuration values are valid before the system starts. + /// + public void Validate() + { + + if (MaxCapacity <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxCapacity), MaxCapacity, "MaxCapacity must be greater than zero."); + } + + if (DeduplicationWindow <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(DeduplicationWindow), DeduplicationWindow, "DeduplicationWindow must be greater than TimeSpan.Zero."); + } + + if (BackpressureRetryDelay <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(BackpressureRetryDelay), BackpressureRetryDelay, "BackpressureRetryDelay must be greater than TimeSpan.Zero."); + } + if (BackpressureRetryDelay > MaxSupportedRetryDelay) + { + throw new ArgumentOutOfRangeException( + nameof(BackpressureRetryDelay), + BackpressureRetryDelay, + $"BackpressureRetryDelay must be less than or equal to {MaxSupportedRetryDelay}."); + } + + if (MaxProcessingAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxProcessingAttempts), MaxProcessingAttempts, "MaxProcessingAttempts must be greater than zero."); + } + + if (MaxDeliveryAttempts <= 0) + { + throw new ArgumentOutOfRangeException(nameof(MaxDeliveryAttempts), MaxDeliveryAttempts, "MaxDeliveryAttempts must be greater than zero."); + } + + if (MaxOutboxRetryAge <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(MaxOutboxRetryAge), MaxOutboxRetryAge, "MaxOutboxRetryAge must be greater than TimeSpan.Zero."); + } + + if (MaxOutboxRetryAge >= DeduplicationWindow) + { + throw new ArgumentOutOfRangeException( + nameof(MaxOutboxRetryAge), + MaxOutboxRetryAge, + "MaxOutboxRetryAge must be less than DeduplicationWindow."); + } + + if (DeadLetterRetentionPeriod <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(DeadLetterRetentionPeriod), + DeadLetterRetentionPeriod, + "DeadLetterRetentionPeriod must be greater than TimeSpan.Zero."); + } + + if (MaxRetainedDeadLetters <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(MaxRetainedDeadLetters), + MaxRetainedDeadLetters, + "MaxRetainedDeadLetters must be greater than zero."); + } + + if (InboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(InboxBatchSize), InboxBatchSize, "InboxBatchSize must be greater than zero."); + } + + if (OutboxBatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(OutboxBatchSize), OutboxBatchSize, "OutboxBatchSize must be greater than zero."); + } + } +} diff --git a/src/Orleans.Messaging/DeliveryResult.cs b/src/Orleans.Messaging/DeliveryResult.cs new file mode 100644 index 00000000000..6124b87548c --- /dev/null +++ b/src/Orleans.Messaging/DeliveryResult.cs @@ -0,0 +1,55 @@ +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Result of attempting to deliver a message to an inbox. +/// +[GenerateSerializer, Alias("Orleans.Messaging.DeliveryResult")] +public readonly struct DeliveryResult +{ + /// + /// The status of the delivery attempt. + /// + [Id(0)] + public DeliveryStatus Status { get; init; } + + /// + /// Optional diagnostic message (e.g., reason for rejection). + /// + [Id(1)] + public string? Message { get; init; } + + /// + /// Creates a result indicating the message was accepted and persisted to inbox. + /// + public static DeliveryResult Accepted() => new() { Status = DeliveryStatus.Accepted }; + + /// + /// Creates a result indicating the message was a duplicate. + /// + public static DeliveryResult Duplicate() => new() { Status = DeliveryStatus.Duplicate }; + + /// + /// Creates a result indicating the inbox is at capacity. + /// + public static DeliveryResult Backpressured() => new() { Status = DeliveryStatus.Backpressured }; + + /// + /// Creates a result indicating the receiving inbox has no registered handler. + /// + public static DeliveryResult HandlerNotFound() => new() + { + Status = DeliveryStatus.HandlerNotFound, + Message = "No inbox handler is registered." + }; + + /// + /// Creates a result indicating the message was dead-lettered. + /// + public static DeliveryResult DeadLettered(string reason) => new() + { + Status = DeliveryStatus.DeadLettered, + Message = reason + }; +} diff --git a/src/Orleans.Messaging/DeliveryStatus.cs b/src/Orleans.Messaging/DeliveryStatus.cs new file mode 100644 index 00000000000..d4d24333e2d --- /dev/null +++ b/src/Orleans.Messaging/DeliveryStatus.cs @@ -0,0 +1,32 @@ +namespace Orleans.Messaging; + +/// +/// Status codes for delivery attempts. +/// +public enum DeliveryStatus +{ + /// + /// Message was accepted and persisted to inbox. + /// + Accepted = 0, + + /// + /// Message was a duplicate (already processed or in inbox). + /// + Duplicate = 1, + + /// + /// Inbox is at capacity; sender should retry later. + /// + Backpressured = 2, + + /// + /// The receiving inbox has no registered handler. + /// + HandlerNotFound = 3, + + /// + /// The message was moved to the receiver's dead-letter store. + /// + DeadLettered = 4 +} diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs new file mode 100644 index 00000000000..628327040ae --- /dev/null +++ b/src/Orleans.Messaging/Envelope.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; + +namespace Orleans.Messaging; + +/// Contains a command identity, payload, subject, and optional application headers. +/// +/// Message identity and payload are required; an empty payload is valid. Raw protocols can omit +/// subject and headers. Prepare the payload and headers before publishing a message and keep them +/// unchanged afterward. Struct assignment shares these objects; ordinary Orleans deep copying +/// and deserialization produce independent payload arrays, header dictionaries, and header value arrays. +/// +[GenerateSerializer, Alias("Orleans.Messaging.Envelope")] +public readonly struct Envelope +{ + /// Gets the exact application-defined command identity within the receiving inbox. + [Id(0)] + public required HierarchicalKey MessageId { get; init; } + + /// Gets the required application payload bytes, including an empty payload. + [Id(1)] + public required byte[] Payload { get; init; } + + /// Gets the optional, nonempty ordinal application protocol subject. + [Id(2)] + public string? Subject { get; init; } + + /// Gets optional application-defined named byte-array headers. + /// Use an ordinal dictionary for case-sensitive header names. Keys are nonempty and arrays are non-null. + [Id(3)] + public Dictionary? Headers { get; init; } +} diff --git a/src/Orleans.Messaging/EnvelopeEquivalence.cs b/src/Orleans.Messaging/EnvelopeEquivalence.cs new file mode 100644 index 00000000000..4710813bacd --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeEquivalence.cs @@ -0,0 +1,17 @@ +using System; + +namespace Orleans.Messaging; + +internal static class EnvelopeEquivalence +{ + public static bool AreEquivalent(OutboxMessage left, OutboxMessage right) => + left.ReceiverId == right.ReceiverId && AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(InboxMessage left, InboxMessage right) => AreSameCommand(left.Envelope, right.Envelope); + + public static bool AreSameCommand(Envelope left, Envelope right) => + left.MessageId == right.MessageId + && left.Payload.Length == right.Payload.Length + && string.Equals(left.Subject, right.Subject, StringComparison.Ordinal) + && left.Payload.AsSpan().SequenceEqual(right.Payload); +} diff --git a/src/Orleans.Messaging/EnvelopeValidation.cs b/src/Orleans.Messaging/EnvelopeValidation.cs new file mode 100644 index 00000000000..e3689b96190 --- /dev/null +++ b/src/Orleans.Messaging/EnvelopeValidation.cs @@ -0,0 +1,49 @@ +using System; +using System.Text; + +namespace Orleans.Messaging; + +internal static class EnvelopeValidation +{ + internal const int MaxMessageIdBytes = 1024; + internal const int MaxSubjectBytes = 256; + private static readonly UTF8Encoding Utf8 = new(false, true); + + // External admission and recovery validate generated values before publishing shared mutations. + public static void Validate(Envelope envelope) + { + ValidateMessageId(envelope.MessageId); + if (envelope.Payload is null) throw new ArgumentException("The payload must not be null.", nameof(envelope)); + if (envelope.Subject is not null) ValidateSubject(envelope.Subject); + if (envelope.Headers is { } headers) + { + foreach (var header in headers) + { + if (string.IsNullOrEmpty(header.Key)) throw new ArgumentException("Header keys must not be empty.", nameof(envelope)); + if (header.Value is null) throw new ArgumentException("Header values must not be null.", nameof(envelope)); + } + } + } + + public static void Validate(InboxMessage message) => Validate(message.Envelope); + + public static void Validate(OutboxMessage message) + { + if (message.ReceiverId.IsDefault) throw new ArgumentException("The outgoing receiver must not be default.", nameof(message)); + Validate(message.Envelope); + } + + internal static void ValidateMessageId(HierarchicalKey messageId) + { + if (messageId.IsDefault) throw new ArgumentException("The message ID must not be unset.", nameof(messageId)); + if (Utf8.GetByteCount(messageId.ToString()) > MaxMessageIdBytes) + throw new ArgumentException($"The message ID exceeds {MaxMessageIdBytes} UTF-8 bytes.", nameof(messageId)); + } + + public static void ValidateSubject(string subject) + { + ArgumentException.ThrowIfNullOrEmpty(subject); + if (Utf8.GetByteCount(subject) > MaxSubjectBytes) + throw new ArgumentException($"The subject exceeds {MaxSubjectBytes} UTF-8 bytes.", nameof(subject)); + } +} diff --git a/src/Orleans.Messaging/HierarchicalKey.cs b/src/Orleans.Messaging/HierarchicalKey.cs new file mode 100644 index 00000000000..a1735eda2ab --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKey.cs @@ -0,0 +1,316 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Orleans.Messaging; + +/// +/// An immutable, ordinal application identity formed from nonempty hierarchical segments. +/// +/// +/// Create constructs literal segments and escapes slash and backslash characters exactly once. +/// Parse reads the canonical escaped path. The key wraps its canonical string; equality and hashing use that string. Applications preserve the identity across retries. +/// +[Immutable, Alias("Orleans.Messaging.HierarchicalKey")] +[JsonConverter(typeof(HierarchicalKeyJsonConverter))] +public readonly struct HierarchicalKey : ISpanFormattable, IEquatable, IParsable, ISpanParsable +{ + /// The escape character used within canonical segments. + public const char EscapeCharacter = '\\'; + + /// The separator between canonical segments. + public const char SegmentSeparator = '/'; + + private readonly string? _value; + + private HierarchicalKey(string canonical) => _value = canonical; + + /// Serializes hierarchical identities as canonical escaped JSON strings. + public sealed class HierarchicalKeyJsonConverter : JsonConverter + { + /// Creates a hierarchical identity JSON converter. + public HierarchicalKeyJsonConverter() { } + + /// + public override HierarchicalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) return default; + if (reader.TokenType is JsonTokenType.String or JsonTokenType.PropertyName + && TryParse(reader.GetString(), null, out var result)) return result; + throw new JsonException("A hierarchical identity must be a canonical escaped path."); + } + + /// + public override void Write(Utf8JsonWriter writer, HierarchicalKey value, JsonSerializerOptions options) + { + if (value.IsDefault) writer.WriteNullValue(); + else writer.WriteStringValue(value.ToString()); + } + + /// + public override HierarchicalKey ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + Read(ref reader, typeToConvert, options); + + /// + public override void WriteAsPropertyName(Utf8JsonWriter writer, HierarchicalKey value, JsonSerializerOptions options) + { + if (value.IsDefault) throw new JsonException("A hierarchical dictionary key must be set."); + writer.WritePropertyName(value.ToString()); + } + } + + /// Gets whether this value is unset. + public bool IsDefault => _value is null; + + /// Gets the canonical path length in UTF-16 characters. + public int Length => _value?.Length ?? 0; + + /// Creates one literal segment, escaping slash and backslash characters. + /// The nonempty literal segment. + /// The segment identity. + /// is null. + /// is empty. + public static HierarchicalKey Create(string value) + { + ArgumentException.ThrowIfNullOrEmpty(value); + return new(Escape(value)); + } + + /// Creates a hierarchy from literal segments in root-first order. + /// The nonempty literal segments. + /// A flat canonical identity which owns its immutable backing string. + /// A segment is null. + /// The input or a segment is empty. + public static HierarchicalKey Create(params ReadOnlySpan values) + { + if (values.IsEmpty) + { + throw new ArgumentException("Values must not be empty.", nameof(values)); + } + var builder = new StringBuilder(); + foreach (var value in values) + { + ArgumentException.ThrowIfNullOrEmpty(value); + if (builder.Length > 0) builder.Append(SegmentSeparator); + AppendEscaped(builder, value); + } + return new(builder.ToString()); + } + + /// Appends one literal child segment. + /// The nonempty literal child segment. + /// The child identity. + /// This key is unset. + /// is null. + /// is empty. + public HierarchicalKey CreateChildKey(string value) + { + EnsureSet(); + ArgumentException.ThrowIfNullOrEmpty(value); + return new(string.Concat(_value, "/", Escape(value))); + } + + /// Composes this hierarchy with an already constructed suffix hierarchy. + /// The constructed suffix. + /// A flat concatenated identity preserving both paths' segment boundaries. + /// This key is unset. + /// is unset. + public HierarchicalKey Append(HierarchicalKey suffix) + { + EnsureSet(); + if (suffix.IsDefault) throw new ArgumentException("The suffix must not be unset.", nameof(suffix)); + return new(string.Concat(_value, "/", suffix._value)); + } + + /// Gets the immediate parent, or null for a root or unset key. + /// The parent identity when this key has more than one segment. + public HierarchicalKey? GetParent() + { + if (_value is null) return null; + var lastSeparator = LastSeparator(_value); + return lastSeparator < 0 ? null : new HierarchicalKey(_value[..lastSeparator]); + } + + /// Tests whether this key is the other's immediate child. + /// The potential parent. + /// Whether there is exactly one additional segment. + public bool IsChildOf(HierarchicalKey other) => other.IsParentOf(this); + + /// Tests whether this key is the other's immediate parent. + /// The potential child. + /// Whether the other key extends this key by exactly one segment. + public bool IsParentOf(HierarchicalKey other) => IsAncestorOf(other) && LastSeparator(other._value!) == Length; + + /// Tests whether this key is equal to or an ancestor of the other key. + /// The identity to inspect. + /// Whether the full prefix consists of equal ordinal segments. Unset keys return false. + public bool IsAncestorOf(HierarchicalKey other) => !IsDefault && !other.IsDefault + && (Equals(other) || other.Length > Length && other._value![Length] == SegmentSeparator + && other._value.StartsWith(_value!, StringComparison.Ordinal)); + + /// + public static HierarchicalKey Parse(string s, IFormatProvider? provider = null) + { + ArgumentNullException.ThrowIfNull(s); + return TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + } + + /// + public static HierarchicalKey Parse(ReadOnlySpan s, IFormatProvider? provider = null) => + TryParse(s, provider, out var result) ? result : throw new FormatException("The value is not a valid canonical hierarchical key."); + + /// + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, out HierarchicalKey result) + { + if (s is not null && IsCanonical(s)) + { + result = new(s); + return true; + } + result = default; + return false; + } + + /// + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out HierarchicalKey result) + { + if (IsCanonical(s)) + { + result = new(new string(s)); + return true; + } + result = default; + return false; + } + + /// + public bool Equals(HierarchicalKey other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + + /// + public override bool Equals(object? obj) => obj is HierarchicalKey other && Equals(other); + + /// + public override int GetHashCode() => _value?.GetHashCode() ?? 0; + + /// Compares complete ordinal key values. + /// The first identity. + /// The second identity. + /// Whether the identities are equal. + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) => left.Equals(right); + + /// Compares complete ordinal key values for inequality. + /// The first identity. + /// The second identity. + /// Whether the identities differ. + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) => !left.Equals(right); + + /// + public override string ToString() => _value ?? string.Empty; + + /// + public string ToString(string? format, IFormatProvider? formatProvider) => ToString(); + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + if (ToString().AsSpan().TryCopyTo(destination)) + { + charsWritten = Length; + return true; + } + charsWritten = 0; + return false; + } + + /// Enumerates escaped canonical segment spans in root-first order. + /// An allocation-free segment enumerator. + public SegmentEnumerator GetEnumerator() => new(ToString().AsSpan()); + + private void EnsureSet() + { + if (IsDefault) throw new InvalidOperationException("The key must not be unset."); + } + + private static string Escape(string value) + { + if (value.AsSpan().IndexOfAny(EscapeCharacter, SegmentSeparator) < 0) return value; + var builder = new StringBuilder(value.Length); + AppendEscaped(builder, value); + return builder.ToString(); + } + + private static void AppendEscaped(StringBuilder builder, string value) + { + foreach (var character in value) + { + if (character is EscapeCharacter or SegmentSeparator) builder.Append(EscapeCharacter); + builder.Append(character); + } + } + + private static int LastSeparator(ReadOnlySpan path) + { + var lastSeparator = -1; + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) i++; + else if (path[i] == SegmentSeparator) lastSeparator = i; + } + return lastSeparator; + } + + private static bool IsCanonical(ReadOnlySpan path) + { + var segmentLength = 0; + for (var i = 0; i < path.Length; i++) + { + if (path[i] == EscapeCharacter) + { + if (++i == path.Length || path[i] is not (EscapeCharacter or SegmentSeparator)) return false; + } + else if (path[i] == SegmentSeparator) + { + if (segmentLength == 0) return false; + segmentLength = 0; + continue; + } + segmentLength++; + } + return segmentLength != 0; + } + + /// Enumerates borrowed spans of the immutable canonical key. + public ref struct SegmentEnumerator + { + private readonly ReadOnlySpan _path; + private int _next; + + internal SegmentEnumerator(ReadOnlySpan path) => _path = path; + + /// Gets the current escaped canonical segment. + public ReadOnlySpan Current { get; private set; } + + /// Advances to the next segment. + /// Whether a segment is available. + public bool MoveNext() + { + if (_next >= _path.Length) + { + Current = default; + return false; + } + var start = _next; + for (; _next < _path.Length; _next++) + { + if (_path[_next] == EscapeCharacter) _next++; + else if (_path[_next] == SegmentSeparator) break; + } + Current = _path[start.._next]; + _next++; + return true; + } + } +} diff --git a/src/Orleans.Messaging/HierarchicalKeyCodec.cs b/src/Orleans.Messaging/HierarchicalKeyCodec.cs new file mode 100644 index 00000000000..98a43ecea81 --- /dev/null +++ b/src/Orleans.Messaging/HierarchicalKeyCodec.cs @@ -0,0 +1,48 @@ +using System; +using System.Buffers; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Cloning; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging; + +[RegisterSerializer] +internal sealed class HierarchicalKeyCodec(IFieldCodec strings) : IFieldCodec +{ + public void WriteField(ref Writer writer, uint fieldIdDelta, + Type? expectedType, HierarchicalKey value) where TBufferWriter : IBufferWriter + { + ReferenceCodec.MarkValueField(writer.Session); + writer.WriteFieldHeader(fieldIdDelta, expectedType, typeof(HierarchicalKey), WireType.TagDelimited); + strings.WriteField(ref writer, 0, typeof(string), value.IsDefault ? null! : value.ToString()); + writer.WriteEndObject(); + } + + public HierarchicalKey ReadValue(ref Reader reader, Field field) + { + field.EnsureWireTypeTagDelimited(); + ReferenceCodec.MarkValueField(reader.Session); + HierarchicalKey result = default; + uint fieldId = 0; + while (true) + { + var header = reader.ReadFieldHeader(); + if (header.IsEndBaseOrEndObject) return result; + fieldId += header.FieldIdDelta; + if (fieldId == 0) + { + var canonical = strings.ReadValue(ref reader, header); + result = canonical is null ? default : HierarchicalKey.Parse(canonical); + } + else reader.ConsumeUnknownField(header); + } + } +} + +[RegisterCopier] +internal sealed class HierarchicalKeyCopier : IDeepCopier +{ + public HierarchicalKey DeepCopy(HierarchicalKey input, CopyContext context) => input; +} diff --git a/src/Orleans.Messaging/Hosting/MessageTypeExtensions.cs b/src/Orleans.Messaging/Hosting/MessageTypeExtensions.cs new file mode 100644 index 00000000000..64344d5740e --- /dev/null +++ b/src/Orleans.Messaging/Hosting/MessageTypeExtensions.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.Messaging; +using Orleans.Serialization; + +namespace Orleans.Hosting; + +/// Extensions for registering typed durable message subjects. +public static class MessageTypeExtensions +{ + /// Registers a singleton subject binding, keyed by its ordinal subject. + /// The subject's payload contract. + /// The service collection. + /// The exact application protocol subject. + /// The service collection. + /// A subject binding is already registered. + public static IServiceCollection AddMessageType(this IServiceCollection services, string subject) + { + ArgumentNullException.ThrowIfNull(services); + EnvelopeValidation.ValidateSubject(subject); + foreach (var descriptor in services) + { + if (descriptor.IsKeyedService + && descriptor.ServiceType.IsGenericType + && descriptor.ServiceType.GetGenericTypeDefinition() == typeof(MessageType<>) + && Equals(descriptor.ServiceKey, subject)) + { + throw new InvalidOperationException($"A payload contract is already registered for durable message subject '{subject}'."); + } + } + + services.TryAddSingleton>(static provider => provider.GetRequiredService().GetSerializer()); + services.AddKeyedSingleton>(subject, + (provider, _) => new MessageType(subject, provider.GetRequiredService>())); + return services; + } +} diff --git a/src/Orleans.Messaging/IInbox.cs b/src/Orleans.Messaging/IInbox.cs new file mode 100644 index 00000000000..840433ba9b2 --- /dev/null +++ b/src/Orleans.Messaging/IInbox.cs @@ -0,0 +1,23 @@ +namespace Orleans.Messaging; + +/// Registers handlers for the messages delivered to a grain's inbox. +/// +/// +/// Each message is dispatched to the first registered handler which can handle it. Handlers can be registered at any +/// time. registers typed subject handlers. +/// +/// +/// A handler decodes the body, performs asynchronous I/O, validates local results, and checks cancellation before +/// the first shared business or journaled mutation. From that mutation until it returns, it executes synchronously: +/// it applies safe-to-commit business changes, stages outgoing messages through , and calls +/// or . The runtime persists the +/// outcome with those changes. Failures before a terminal operation follow the inbox retry and dead-letter policy. +/// +/// +public interface IInbox +{ + /// Registers a handler. + /// The handler. + /// This inbox. + IInbox Register(IInboxHandler handler); +} diff --git a/src/Orleans.Messaging/IInboxExtension.cs b/src/Orleans.Messaging/IInboxExtension.cs new file mode 100644 index 00000000000..0d83a287783 --- /dev/null +++ b/src/Orleans.Messaging/IInboxExtension.cs @@ -0,0 +1,33 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans; +using Orleans.Runtime; +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Non-generic grain extension for inbox message delivery. +/// +[Alias("IInboxExtension")] +public interface IInboxExtension : IGrainExtension +{ + /// + /// Delivers a message to this grain's inbox. + /// + /// The received command with a required identity and payload; its receiver is this grain. + /// Cancels the caller's wait for delivery. + /// + /// Direct admission shares the payload array, header dictionary, and header value arrays under the immutable-publication contract. + /// RPC copying and deserialization isolate payload arrays, header dictionaries, and header value arrays using ordinary Orleans serialization. + /// A successful result means the inbox durably accepted the message, or already processed it. + /// + /// Result indicating delivery/processing status. + /// + /// has an unset message ID, exceeds identity admission limits, or has invalid payload or headers. + /// Optional standard metadata is validated where its contract is consumed. + /// + [Alias("DeliverAsync")] + ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default); +} diff --git a/src/Orleans.Messaging/IInboxHandler.cs b/src/Orleans.Messaging/IInboxHandler.cs new file mode 100644 index 00000000000..843d665ecfe --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandler.cs @@ -0,0 +1,23 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// Handles inbox messages which it accepts. +/// +/// The inbox dispatches each message to the first registered handler whose returns +/// . +/// +public interface IInboxHandler +{ + /// Determines whether this handler handles the message. + /// The message and its terminal operations. + /// when this handler handles the message. + bool CanHandle(IInboxHandlerContext context); + + /// Handles the message. + /// The message and its terminal operations. + /// The attempt token. + /// A task which completes when handling returns. + ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); +} diff --git a/src/Orleans.Messaging/IInboxHandlerContext.cs b/src/Orleans.Messaging/IInboxHandlerContext.cs new file mode 100644 index 00000000000..dd37dedffc0 --- /dev/null +++ b/src/Orleans.Messaging/IInboxHandlerContext.cs @@ -0,0 +1,52 @@ +namespace Orleans.Messaging; + +/// +/// Exposes the received message and its attempt-scoped terminal outcome. +/// +public interface IInboxHandlerContext +{ + /// + /// Gets the received message and its immutable application headers and payload. + /// + /// + /// The context keeps the message available through actual handler completion, including + /// after Complete or Fail removes the pending message. Keep its payload and headers unchanged while handling the command. + /// + InboxMessage Message { get; } + + /// + /// Synchronously stages inbox completion and transport deduplication for the active attempt. + /// + /// + /// Apply safe-to-commit business mutations and stage outgoing messages before calling Complete + /// in the same synchronous final block. Return from the handler without further awaits. + /// The runtime owns the subsequent journal write and acknowledgement. + /// Repeated completion within the same active attempt coalesces. Completion retains its + /// logical outcome when cancellation arrives after the final block starts. + /// + /// + /// The context is retired or belongs to another activation or handler attempt. + /// + void Complete(); + + /// + /// Synchronously stages a permanent processing failure, retaining the message as a dead letter + /// and recording its identity for transport deduplication. + /// + /// The nonblank diagnostic reason for the permanent failure. + /// + /// Stage any safe-to-commit business changes and outgoing messages before this operation, + /// then return synchronously. The runtime persists the dead letter and completion together + /// and owns their journal acknowledgement. The dead letter records the current attempt count. + /// Repeated calls with the same reason coalesce; Complete preserves an already staged dead letter. + /// Cancellation after staging preserves the terminal outcome. + /// Use an application reply and Complete for normal business rejection outcomes. + /// + /// The reason is blank. + /// The reason is null. + /// + /// The context is retired or belongs to another activation or attempt, or a different + /// terminal outcome has already been staged. + /// + void Fail(string reason); +} diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs new file mode 100644 index 00000000000..2103b548fff --- /dev/null +++ b/src/Orleans.Messaging/IOutbox.cs @@ -0,0 +1,36 @@ +using System; + +namespace Orleans.Messaging; + +/// +/// Stages opaque durable messages alongside a grain's journaled business state. +/// +/// +/// The outbox state machine's asynchronous write preparation establishes a durable self-wakeup before capture. +/// Its write-completion callback makes captured messages eligible for dispatch. +/// Each wakeup checks the grain's messages and their retry deadlines. Messages remain pending until the destination +/// acknowledges durable acceptance, recognizes a duplicate, or reports a terminal delivery outcome. +/// Applications namespace command identities and define ordering in their protocols. +/// +public interface IOutbox +{ + /// + /// Synchronously stages an outgoing command for the grain's next journal write. + /// + /// The fully built outgoing message and required receiver. + /// + /// Direct sends share the payload array, header dictionary, and header value arrays. Keep these objects unchanged after publication. + /// Equivalent repeated identities retain the original message and headers; payload, optional subject, + /// and destination must match. Custom headers do not change pending intent equality. Conflicts fail explicitly. + /// Inbox handlers stage outgoing + /// messages in their synchronous final block before calling + /// or . + /// Ordinary callers persist staged messages using their journaled state manager. + /// An explicit write retry retains pending business changes and messages after a scheduling failure. + /// + /// The envelope has an unset identity, invalid subject or destination, or exceeds identity/subject limits. + /// + /// The identity conflicts, or the outbox is unavailable. + /// + void Send(OutboxMessage message); +} diff --git a/src/Orleans.Messaging/InboxDispatcher.cs b/src/Orleans.Messaging/InboxDispatcher.cs new file mode 100644 index 00000000000..3eea4acfb3a --- /dev/null +++ b/src/Orleans.Messaging/InboxDispatcher.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// Dispatches each inbox message to the first registered handler which can handle it. +internal readonly struct InboxDispatcher +{ + private readonly List _handlers; + + internal InboxDispatcher(List handlers) => _handlers = handlers; + + internal bool IsEmpty => _handlers.Count == 0; + + internal void Add(IInboxHandler handler) + { + ArgumentNullException.ThrowIfNull(handler); + _handlers.Add(handler); + } + + internal ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var handler in _handlers) + { + if (handler.CanHandle(context)) + { + return handler.HandleAsync(context, cancellationToken); + } + } + + throw new InvalidOperationException( + $"No handler is registered for message subject '{context.Message.Envelope.Subject}'."); + } +} diff --git a/src/Orleans.Messaging/InboxExtensions.cs b/src/Orleans.Messaging/InboxExtensions.cs new file mode 100644 index 00000000000..18eb0af1fa6 --- /dev/null +++ b/src/Orleans.Messaging/InboxExtensions.cs @@ -0,0 +1,127 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Messaging; + +/// Registers typed subject handlers with an . +/// Stateful overloads store their argument directly and support static delegates. +public static class InboxExtensions +{ + /// Registers a synchronous subject handler. + /// The body contract. + /// The inbox. + /// The subject and serializer binding. + /// The synchronous body and context delegate. + /// The inbox. + public static IInbox Register(this IInbox inbox, MessageType messageType, Action handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new SynchronousHandler(messageType, handler)); + } + + /// Registers a synchronous subject handler with an explicit argument. + /// The body contract. + /// The handler argument type. + /// The inbox. + /// The subject and serializer binding. + /// The argument stored with the handler. + /// The synchronous body, argument, and context delegate. + /// The inbox. + /// Use a static delegate to pass state without a captured closure. + public static IInbox Register( + this IInbox inbox, MessageType messageType, TArg argument, Action handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new SynchronousHandler(messageType, argument, handler)); + } + + /// Registers a subject handler supporting asynchronous preparation. + /// The body contract. + /// The inbox. + /// The subject and serializer binding. + /// The body, context, and attempt-token delegate. + /// The inbox. + public static IInbox Register( + this IInbox inbox, MessageType messageType, Func handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new AsynchronousHandler(messageType, handler)); + } + + /// Registers an asynchronous subject handler with an explicit argument. + /// The body contract. + /// The handler argument type. + /// The inbox. + /// The subject and serializer binding. + /// The argument stored with the handler. + /// The body, argument, context, and attempt-token delegate. + /// The inbox. + /// Use a static delegate to pass state without a captured closure. + public static IInbox Register( + this IInbox inbox, MessageType messageType, TArg argument, + Func handler) + { + ArgumentNullException.ThrowIfNull(inbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(handler); + return inbox.Register(new AsynchronousHandler(messageType, argument, handler)); + } + + private abstract class Handler(MessageType messageType) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) => + string.Equals(context.Message.Envelope.Subject, messageType.Subject, StringComparison.Ordinal); + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + var body = messageType.Decode(context.Message); + cancellationToken.ThrowIfCancellationRequested(); + return Invoke(body, context, cancellationToken); + } + + protected abstract ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken); + } + + private sealed class SynchronousHandler( + MessageType messageType, Action handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) + { + handler(body, context); + return ValueTask.CompletedTask; + } + } + + private sealed class SynchronousHandler( + MessageType messageType, TArg argument, Action handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) + { + handler(body, argument, context); + return ValueTask.CompletedTask; + } + } + + private sealed class AsynchronousHandler( + MessageType messageType, + Func handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) => + handler(body, context, cancellationToken); + } + + private sealed class AsynchronousHandler( + MessageType messageType, TArg argument, + Func handler) : Handler(messageType) + { + protected override ValueTask Invoke(T body, IInboxHandlerContext context, CancellationToken cancellationToken) => + handler(body, argument, context, cancellationToken); + } +} diff --git a/src/Orleans.Messaging/InboxMessage.cs b/src/Orleans.Messaging/InboxMessage.cs new file mode 100644 index 00000000000..b2d6b6004ca --- /dev/null +++ b/src/Orleans.Messaging/InboxMessage.cs @@ -0,0 +1,10 @@ +namespace Orleans.Messaging; + +/// A received command whose destination is the grain owning the inbox. +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessage")] +public readonly struct InboxMessage +{ + /// Gets the command identity, payload, and application metadata. + [Id(0)] + public required Envelope Envelope { get; init; } +} diff --git a/src/Orleans.Messaging/MessageType.cs b/src/Orleans.Messaging/MessageType.cs new file mode 100644 index 00000000000..0a895921ea1 --- /dev/null +++ b/src/Orleans.Messaging/MessageType.cs @@ -0,0 +1,93 @@ +using System; +using Orleans.Runtime; +using Orleans.Serialization; + +namespace Orleans.Messaging; + +/// +/// Binds an exact message subject to an Orleans payload serializer. +/// +/// The subject's payload contract. +/// +/// Bindings are immutable and can be shared across activations. Decode during handler preparation; +/// decoded resource-bearing values retain their ordinary application-managed ownership. +/// +public sealed class MessageType +{ + private readonly Serializer _serializer; + + /// Creates a subject and payload-type binding. + /// The ordinal protocol subject. + /// The payload serializer. + public MessageType(string subject, Serializer serializer) + { + EnvelopeValidation.ValidateSubject(subject); + ArgumentNullException.ThrowIfNull(serializer); + Subject = subject; + _serializer = serializer; + } + + /// Gets the exact protocol subject. + public string Subject { get; } + + /// Decodes an envelope using this subject's payload contract. + /// The envelope containing the serialized payload. + /// The nonnull decoded payload. + /// The subject is absent or different, or the payload is a serialized null value. + public T Decode(Envelope envelope) + { + if (envelope.Subject is null) + { + throw new ArgumentException( + $"Message subject '{Subject}' is required.", nameof(envelope)); + } + if (!string.Equals(Subject, envelope.Subject, StringComparison.Ordinal)) + { + throw new ArgumentException( + $"Expected message subject '{Subject}', but received '{envelope.Subject}'.", + nameof(envelope)); + } + + return _serializer.Deserialize(envelope.Payload) + ?? throw new ArgumentException($"Message subject '{Subject}' requires a nonnull payload.", nameof(envelope)); + } + + /// Decodes the payload of a received message. + /// The received message. + /// The nonnull decoded payload. + public T Decode(InboxMessage message) => Decode(message.Envelope); + + /// Decodes the payload of an outgoing message. + /// The outgoing message. + /// The nonnull decoded payload. + public T Decode(OutboxMessage message) => Decode(message.Envelope); + + /// Encodes a typed body into an outgoing message. + /// The stable application command identity. + /// The destination inbox. + /// The nonnull body to serialize. + /// An outgoing message with the encoded payload. + /// + /// Serialization completes before publication. Keep the published payload bytes unchanged. + /// + public OutboxMessage Create(HierarchicalKey messageId, GrainId receiverId, T body) + { + if (body is null) throw new ArgumentNullException(nameof(body)); + if (receiverId.IsDefault) + { + throw new ArgumentException("The receiving grain ID must be set.", nameof(receiverId)); + } + EnvelopeValidation.ValidateMessageId(messageId); + var payload = _serializer.SerializeToArray(body); + return new OutboxMessage + { + Envelope = new Envelope + { + MessageId = messageId, + Subject = Subject, + Payload = payload + }, + ReceiverId = receiverId + }; + } +} diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj new file mode 100644 index 00000000000..69982a6b231 --- /dev/null +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -0,0 +1,31 @@ + + + Microsoft.Orleans.Messaging + Microsoft Orleans Messaging + Inbox and outbox messaging for Microsoft Orleans. + $(PackageTags) Messaging Inbox Outbox + true + $(DefaultTargetFrameworks) + disable + enable + $(NoWarn);ORLEANSEXP005 + $(VersionSuffix).alpha.1 + alpha.1 + + + true + + + + + + + + + + + + + + + diff --git a/src/Orleans.Messaging/OrleansContracts.txt b/src/Orleans.Messaging/OrleansContracts.txt new file mode 100644 index 00000000000..4a090bf615d --- /dev/null +++ b/src/Orleans.Messaging/OrleansContracts.txt @@ -0,0 +1,14 @@ +# This file is generated by the Orleans contract analyzer. +# To regenerate this project from the repository root: +# dotnet format PATH_TO_PROJECT.csproj analyzers --severity info --diagnostics ORLEANS0016 ORLEANS0017 ORLEANS0018 ORLEANS0019 ORLEANS0020 ORLEANS0022 ORLEANS0023 ORLEANS0024 +# Run the command once per contract project; do not pass a .sln or .slnx path. +# Verify with: dotnet build PATH_TO_PROJECT.csproj +# The regeneration command edits this manifest only; it does not change source attributes. +# OrleansContracts format: 2 +# Method lines use: wire-identity: CLR-signature. +# The identity is the identifier Orleans uses at runtime, whether generated or declared in source. +# Review every diff: identity or signature changes can break wire compatibility during rolling upgrades. +# Details: https://aka.ms/orleans/OrleansContracts.txt + +interface [GrainInterfaceType("Orleans.Messaging.IInboxExtension")] Orleans.Messaging.IInboxExtension [Version(0)] + DeliverAsync: DeliverAsync(Orleans.Messaging.InboxMessage, System.Threading.CancellationToken) -> ValueTask diff --git a/src/Orleans.Messaging/OutboxExtensions.cs b/src/Orleans.Messaging/OutboxExtensions.cs new file mode 100644 index 00000000000..da7a623d6e8 --- /dev/null +++ b/src/Orleans.Messaging/OutboxExtensions.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// Extensions for encoding and staging typed messages and replies. +public static class OutboxExtensions +{ + /// Encodes and synchronously stages one typed command. + /// The body contract. + /// The owning grain's outbox. + /// The subject and serializer binding. + /// The stable application command identity. + /// The destination inbox. + /// The body to encode. + /// + /// Serialization finishes before staging begins. Compute fallible business results first, + /// then send, apply the business state, and complete in one synchronous final block. + /// Payload and header arrays remain available through the staged message's ordinary managed references. + /// + public static void Send( + this IOutbox outbox, MessageType messageType, + HierarchicalKey messageId, GrainId destination, T body) + { + ArgumentNullException.ThrowIfNull(outbox); + ArgumentNullException.ThrowIfNull(messageType); + var message = messageType.Create(messageId, destination, body); + outbox.Send(message); + } + + /// Encodes and synchronously stages one typed command to a grain reference. + /// The body contract. + /// The owning grain's outbox. + /// The subject and serializer binding. + /// The stable application command identity. + /// The destination grain. + /// The body to encode. + public static void Send( + this IOutbox outbox, MessageType messageType, + HierarchicalKey messageId, IAddressable destination, T body) + { + ArgumentNullException.ThrowIfNull(destination); + outbox.Send(messageType, messageId, destination.GetGrainId(), body); + } + + /// Encodes and synchronously stages typed commands in enumeration order. + /// The body contract. + /// The owning grain's outbox. + /// The subject and serializer binding. + /// The application-defined identities, destinations, and bodies. + /// + /// Enumerates once and sends each command using the ordinary typed send helper. + /// If enumeration, encoding, or staging fails, earlier messages remain staged + /// under their original identities with their encoded payload and headers. + /// Retry with the same identities and content; persistence follows the grain's + /// ordinary journal write. + /// + public static void Send( + this IOutbox outbox, MessageType messageType, + IEnumerable<(HierarchicalKey MessageId, GrainId Destination, T Body)> messages) + { + ArgumentNullException.ThrowIfNull(outbox); + ArgumentNullException.ThrowIfNull(messageType); + ArgumentNullException.ThrowIfNull(messages); + foreach (var (messageId, destination, body) in messages) + { + outbox.Send(messageType, messageId, destination, body); + } + } + + /// Encodes and stages a reply under the received command's deterministic result child. + /// The reply body contract. + /// The replying grain's outbox. + /// The reply subject and serializer binding. + /// The current received message and completion context. + /// The application-selected reply destination. + /// The reply body. + /// The reply identity is context.Message.Envelope.MessageId.CreateChildKey("result"). + public static void SendReply( + this IOutbox outbox, MessageType messageType, + IInboxHandlerContext context, GrainId destination, T body) + { + ArgumentNullException.ThrowIfNull(context); + outbox.Send(messageType, context.Message.Envelope.MessageId.CreateChildKey("result"), destination, body); + } + + /// Encodes and stages a deterministic result reply to a grain reference. + /// The reply body contract. + /// The replying grain's outbox. + /// The reply subject and serializer binding. + /// The current received message and completion context. + /// The application-selected replying grain reference. + /// The reply body. + public static void SendReply( + this IOutbox outbox, MessageType messageType, + IInboxHandlerContext context, IAddressable destination, T body) + { + ArgumentNullException.ThrowIfNull(destination); + outbox.SendReply(messageType, context, destination.GetGrainId(), body); + } +} diff --git a/src/Orleans.Messaging/OutboxMessage.cs b/src/Orleans.Messaging/OutboxMessage.cs new file mode 100644 index 00000000000..4a4ee561998 --- /dev/null +++ b/src/Orleans.Messaging/OutboxMessage.cs @@ -0,0 +1,20 @@ +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// An outgoing command with destination routing separate from its common envelope. +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessage")] +public readonly struct OutboxMessage +{ + /// Gets the command identity, payload, and application metadata. + [Id(0)] + public required Envelope Envelope { get; init; } + + /// Gets the required destination grain identity. + [Id(1)] + public required GrainId ReceiverId { get; init; } + + /// Creates the incoming view by sharing the envelope, payload, and headers. + /// A received message whose destination is implied by the receiving grain. + public InboxMessage ToInboxMessage() => new() { Envelope = Envelope }; +} diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md new file mode 100644 index 00000000000..3e34b3285d4 --- /dev/null +++ b/src/Orleans.Messaging/README.md @@ -0,0 +1,77 @@ +# Microsoft Orleans Messaging + +`Microsoft.Orleans.Messaging` provides grain-scoped durable inboxes and outboxes +for Microsoft Orleans. A grain sends a message by staging it in its outbox; the +receiver's inbox processes it at least once and deduplicates it by message ID. + +This package defines the messaging contracts: envelopes, hierarchical message +identities, inbox and outbox interfaces, handler registration, typed message +bindings, delivery results, and options. + +## Messages + +`Envelope` carries: + +- `MessageId`: a `HierarchicalKey` supplied by the application, at most 1,024 UTF-8 + bytes. It identifies one command at the receiver. +- `Payload`: opaque bytes; an empty payload is valid. +- `Subject`: an optional ordinal name, at most 256 UTF-8 bytes, that selects a typed + handler. +- `Headers`: optional named byte arrays with nonempty keys and non-null values. + +`OutboxMessage` pairs an envelope with its receiver. `InboxMessage` carries the same +envelope at the receiver. Keep published payload and header arrays unchanged. + +`HierarchicalKey` wraps its canonical escaped path. Equality and hashing compare that +path ordinally. Use `Create`, `CreateChildKey`, and `Append` to build keys, and +`GetParent`, `IsParentOf`, `IsChildOf`, `IsAncestorOf`, and the segment enumerator to +navigate them. + +## Handle messages + +`IInbox.Register(IInboxHandler)` adds a handler. Each message goes to the first +registered handler whose `CanHandle` returns `true`. + +`InboxExtensions.Register` binds a `MessageType` subject to a typed method: + +```csharp +inbox.Register(reserve, HandleReserve); +inbox.Register(reserve, this, static (request, grain, context) => grain.HandleReserve(request, context)); +``` + +Overloads accept synchronous or asynchronous delegates, with or without a `TArg` +argument for static, allocation-free delegates. Register typed subjects with +`AddMessageType(subject)`. + +A handler finishes each message with `IInboxHandlerContext.Complete()` or +`Fail(reason)`: + +- `Complete()` stages the message's removal and completion record. +- `Fail(reason)` moves the message to the inbox dead letters and records its ID as + processed. + +## Send messages + +`IOutbox.Send(OutboxMessage)` stages a message. `OutboxExtensions` adds typed helpers: + +- `Send(type, messageId, receiver, body)` encodes and stages one message; a batch + overload stages several. +- `SendReply(type, context, destination, body)` stages a reply whose ID is the + received message ID's `result` child, sent to a destination the application + carries in its request. + +`MessageType.Create` and `OutboxMessage.ToInboxMessage()` build messages for direct +inbox delivery. + +## Delivery results + +Each receiver returns a `DeliveryResult`: + +- `Accepted`: the message is durable in the inbox. +- `Duplicate`: the message ID is pending or completed within `DeduplicationWindow`. +- `Backpressured`: the inbox holds `MaxCapacity` pending messages. +- `HandlerNotFound`: the receiver has no registered handlers. +- `DeadLettered`: the message moved to the receiver's dead letters. + +`InboxOptions` configures capacity, batch sizes, retries, deduplication, and +dead-letter retention. diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs new file mode 100644 index 00000000000..16cd4a72d6d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -0,0 +1,103 @@ +using Orleans.Messaging.Configuration; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DeliveryAndOptionsContractTests +{ + [Fact] + public void DeliveryResult_EachFactory_PreservesStatusAndPayload() + { + var handlerMissing = DeliveryResult.HandlerNotFound(); + var deadLettered = DeliveryResult.DeadLettered("poison body"); + + Assert.Equal(DeliveryStatus.Accepted, DeliveryResult.Accepted().Status); + Assert.Equal(DeliveryStatus.Duplicate, DeliveryResult.Duplicate().Status); + Assert.Equal(DeliveryStatus.Backpressured, DeliveryResult.Backpressured().Status); + Assert.Equal(DeliveryStatus.HandlerNotFound, handlerMissing.Status); + Assert.Equal("No inbox handler is registered.", handlerMissing.Message); + Assert.Equal(DeliveryStatus.DeadLettered, deadLettered.Status); + Assert.Equal("poison body", deadLettered.Message); + } + + [Fact] + public void DeliveryStatus_AllValues_HaveContiguousDistinctValues() + { + Assert.Equal( + [ + DeliveryStatus.Accepted, + DeliveryStatus.Duplicate, + DeliveryStatus.Backpressured, + DeliveryStatus.HandlerNotFound, + DeliveryStatus.DeadLettered + ], + Enum.GetValues()); + Assert.Equal([0, 1, 2, 3, 4], Enum.GetValues().Select(static value => (int)value)); + } + + [Fact] + public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() + { + var options = new InboxOptions(); + + options.Validate(); + + Assert.Equal(1000, options.MaxCapacity); + Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); + Assert.Equal(TimeSpan.FromDays(1), options.MaxOutboxRetryAge); + Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); + Assert.Equal(5, options.MaxProcessingAttempts); + Assert.Equal(100, options.MaxDeliveryAttempts); + Assert.Equal(1000, options.MaxRetainedDeadLetters); + Assert.Equal(32, options.InboxBatchSize); + Assert.Equal(32, options.OutboxBatchSize); + } + + [Fact] + public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContract() + { + var invalidCases = new (string Parameter, Action Mutate)[] + { + (nameof(InboxOptions.MaxCapacity), options => options.MaxCapacity = 0), + (nameof(InboxOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), + (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), + (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), + (nameof(InboxOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), + (nameof(InboxOptions.MaxDeliveryAttempts), options => options.MaxDeliveryAttempts = 0), + (nameof(InboxOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), + (nameof(InboxOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), + (nameof(InboxOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), + (nameof(InboxOptions.InboxBatchSize), options => options.InboxBatchSize = 0), + (nameof(InboxOptions.OutboxBatchSize), options => options.OutboxBatchSize = 0), + }; + + foreach (var (parameter, mutate) in invalidCases) + { + var options = new InboxOptions(); + mutate(options); + var exception = Assert.Throws(options.Validate); + Assert.Equal(parameter, exception.ParamName); + } + } + + + [Fact] + public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() + { + foreach (var retryAge in new[] { TimeSpan.FromHours(2), TimeSpan.FromHours(3) }) + { + var options = new InboxOptions + { + DeduplicationWindow = TimeSpan.FromHours(2), + MaxOutboxRetryAge = retryAge, + }; + + var exception = Assert.Throws(options.Validate); + Assert.Equal(nameof(InboxOptions.MaxOutboxRetryAge), exception.ParamName); + Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs new file mode 100644 index 00000000000..cb017e464db --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -0,0 +1,315 @@ +using System.Reflection; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EnvelopeContractTests +{ + private static HierarchicalKey Id => HierarchicalKey.Create("orders", "123"); + private static GrainId Receiver => GrainId.Create("receiver", "123"); + + [Fact] + public void RequiredPayloadAndOptionalMetadata() + { + var envelope = Create([]); + Validate(envelope); + Validate(new InboxMessage { Envelope = envelope }); + Assert.Equal(Id, envelope.MessageId); + Assert.Empty(envelope.Payload); + Assert.Null(envelope.Subject); + Assert.Null(envelope.Headers); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var decoded = serializer.Deserialize(serializer.SerializeToArray(envelope)); + Validate(decoded); + Assert.Empty(decoded.Payload); + Assert.Null(decoded.Subject); + Assert.Null(decoded.Headers); + Assert.Throws(() => Validate(new OutboxMessage { Envelope = envelope, ReceiverId = default })); + Assert.Throws(() => Validate(default(Envelope))); + Assert.Throws(() => Validate(envelope with { Payload = null! })); + } + + [Fact] + public void FirstClassSubjectAndEncodedSenderUseByteHeaders() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var senderSerializer = services.GetRequiredService>(); + var sender = GrainId.Create("sender", "123"); + var envelope = Create([1]) with + { + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) + { + ["subject"] = "extra subject"u8.ToArray(), + ["sender"] = senderSerializer.SerializeToArray(sender), + ["custom"] = "first"u8.ToArray(), + ["CUSTOM"] = "second"u8.ToArray() + } + }; + Validate(envelope); + Assert.Equal("orders.v1", envelope.Subject); + Assert.Equal(sender, senderSerializer.Deserialize(envelope.Headers["sender"])); + Assert.Equal("extra subject"u8.ToArray(), envelope.Headers["subject"]); + Assert.Equal("first"u8.ToArray(), envelope.Headers["custom"]); + Assert.Equal("second"u8.ToArray(), envelope.Headers["CUSTOM"]); + Assert.False(envelope.Headers.TryAdd("custom", "replacement"u8.ToArray())); + Assert.Equal("first"u8.ToArray(), envelope.Headers["custom"]); + var serializer = services.GetRequiredService>(); + var decoded = serializer.Deserialize(serializer.SerializeToArray(envelope)); + Assert.NotNull(decoded.Headers); + Assert.Equal("orders.v1", decoded.Subject); + Assert.Equal(sender, senderSerializer.Deserialize(decoded.Headers["sender"])); + Assert.NotSame(envelope.Headers["sender"], decoded.Headers["sender"]); + Assert.Equal("extra subject"u8.ToArray(), decoded.Headers["subject"]); + } + + [Fact] + public void StructAssignmentAndLocalConversionSharePublishedObjects() + { + var envelope = Create([0, 255, 128]) with { Headers = new(StringComparer.Ordinal) { ["trace"] = "original"u8.ToArray() } }; + var assigned = envelope; + var outgoing = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + var incoming = outgoing.ToInboxMessage(); + Assert.Same(envelope.Payload, assigned.Payload); + Assert.Same(envelope.Headers, assigned.Headers); + Assert.Same(envelope.Payload, incoming.Envelope.Payload); + Assert.Same(envelope.Headers, incoming.Envelope.Headers); + Assert.Equal(Id, incoming.Envelope.MessageId); + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Null(typeof(Envelope).GetProperty("ReceiverId")); + } + + [Theory] + [InlineData(0)] + [InlineData(7)] + [InlineData(100000)] + public void OrdinaryCopyAndIndependentWireRoundTrip(int length) + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var payload = Enumerable.Range(0, length).Select(i => (byte)i).ToArray(); + var envelope = Create(payload) with + { + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) { ["custom"] = "retained"u8.ToArray(), ["CUSTOM"] = "distinct"u8.ToArray(), ["empty"] = [] } + }; + var serializer = sending.GetRequiredService>(); + var wire = serializer.SerializeToArray(envelope); + var decoded = receiving.GetRequiredService>().Deserialize(wire); + var copied = sending.GetRequiredService().Copy(envelope); + foreach (var actual in new[] { decoded, copied }) + { + Validate(actual); + Assert.Equal(Id, actual.MessageId); + Assert.Equal(payload, actual.Payload); + Assert.Equal("orders.v1", actual.Subject); + Assert.NotNull(actual.Headers); + Assert.NotSame(envelope.Headers, actual.Headers); + Assert.Equal("retained"u8.ToArray(), actual.Headers["custom"]); + Assert.Equal("distinct"u8.ToArray(), actual.Headers["CUSTOM"]); + Assert.Empty(actual.Headers["empty"]); + Assert.NotSame(envelope.Headers["custom"], actual.Headers["custom"]); + actual.Headers["custom"][0] = 0; + Assert.Equal("retained"u8.ToArray(), envelope.Headers["custom"]); + if (length > 0) + { + Assert.NotSame(envelope.Payload, actual.Payload); + actual.Payload[0] = 42; + Assert.Equal((byte)0, envelope.Payload[0]); + } + } + Assert.Equal(wire, serializer.SerializeToArray(envelope)); + } + + [Fact] + public void PayloadUsesOrdinaryExternallyBoundOrleansSerializer() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var envelope = Create(serializer.SerializeToArray("typed application payload")) with { Subject = "typed.v1" }; + Assert.Equal("typed application payload", serializer.Deserialize(envelope.Payload)); + Assert.Equal("typed.v1", envelope.Subject); + Assert.Null(envelope.Headers); + } + + [Theory] + [InlineData("InboxMessage")] + [InlineData("OutboxMessage")] + public void MessageAndDeadLetterCopyPreservesMetadataAndIsolation(string kind) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var envelope = Create([0, 255]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "value"u8.ToArray() } }; + object value = kind.StartsWith("Inbox", StringComparison.Ordinal) ? new InboxMessage { Envelope = envelope } + : new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) + { + var type = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + kind, true)!; + var record = Activator.CreateInstance(type)!; + type.GetProperty("Message")!.SetValue(record, value); + type.GetProperty("DeadLetteredAt")!.SetValue(record, DateTimeOffset.UnixEpoch); + type.GetProperty("Reason")!.SetValue(record, "failure"); + type.GetProperty("AttemptCount")!.SetValue(record, 3); + value = record; + } + var serializer = services.GetRequiredService(); + foreach (var result in new[] { serializer.Deserialize(serializer.SerializeToArray(value)), services.GetRequiredService().Copy(value) }) + { + Assert.NotNull(result); + var message = kind.EndsWith("DeadLetter", StringComparison.Ordinal) ? result.GetType().GetProperty("Message")!.GetValue(result)! : result; + var actual = message is InboxMessage incoming ? incoming.Envelope : Assert.IsType(message).Envelope; + Assert.Equal(new byte[] { 0, 255 }, actual.Payload); + Assert.NotSame(envelope.Payload, actual.Payload); + Assert.Equal("subject.v1", actual.Subject); + Assert.NotNull(actual.Headers); + Assert.NotSame(envelope.Headers, actual.Headers); + Assert.Equal("value"u8.ToArray(), actual.Headers["custom"]); + Assert.NotSame(envelope.Headers!["custom"], actual.Headers["custom"]); + if (message is OutboxMessage outgoing) Assert.Equal(Receiver, outgoing.ReceiverId); + if (kind.EndsWith("DeadLetter", StringComparison.Ordinal)) Assert.Equal(3, result.GetType().GetProperty("AttemptCount")!.GetValue(result)); + } + } + + [Fact] + public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() + { + AssertIds(typeof(Envelope), ("MessageId", 0u), ("Payload", 1u), ("Subject", 2u), ("Headers", 3u)); + AssertIds(typeof(InboxMessage), ("Envelope", 0u)); + AssertIds(typeof(OutboxMessage), ("Envelope", 0u), ("ReceiverId", 1u)); + Assert.Equal(typeof(Dictionary), typeof(Envelope).GetProperty("Headers")!.PropertyType); + Assert.Equal(typeof(byte[]), typeof(Envelope).GetProperty("Payload")!.PropertyType); + Assert.Null(typeof(Envelope).GetCustomAttribute()); + Assert.Single(typeof(Envelope).GetCustomAttributes()); + Assert.Equal("Orleans.Messaging.Envelope", typeof(Envelope).GetCustomAttribute()!.Alias); + Assert.Equal(new[] { "Headers", "MessageId", "Payload", "Subject" }, typeof(Envelope).GetProperties().Select(p => p.Name).Order(StringComparer.Ordinal)); + Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.PackedEnvelopeHeaders")); + Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.EnvelopeBuilder")); + Assert.Equal(typeof(void), typeof(IInboxHandlerContext).GetMethod("Fail")!.ReturnType); + Assert.Equal(typeof(string), Assert.Single(typeof(IInboxHandlerContext).GetMethod("Fail")!.GetParameters()).ParameterType); + Assert.Equal(new[] { "Accepted:0", "Backpressured:2", "DeadLettered:4", "Duplicate:1", "HandlerNotFound:3" }, + Enum.GetNames().Select(name => $"{name}:{(int)Enum.Parse(name)}").Order(StringComparer.Ordinal)); + AssertIds(typeof(DeliveryResult), ("Status", 0u), ("Message", 1u)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var outgoing = new OutboxMessage { Envelope = Create([7]), ReceiverId = Receiver }; + var other = outgoing with { ReceiverId = GrainId.Create("receiver", "other") }; + var serializer = services.GetRequiredService>(); + Assert.Equal(serializer.SerializeToArray(outgoing.ToInboxMessage()), serializer.SerializeToArray(other.ToInboxMessage())); + } + + [Theory] + [InlineData("same", true, true)] + [InlineData("headers", true, true)] + [InlineData("sender", true, true)] + [InlineData("receiver", false, true)] + [InlineData("id", false, false)] + [InlineData("child", false, false)] + [InlineData("subject", false, false)] + [InlineData("subject-case", false, false)] + [InlineData("no-subject", false, false)] + [InlineData("payload", false, false)] + [InlineData("payload-length", false, false)] + public void PendingCommandEqualityIgnoresExtraHeadersAndSender(string change, bool outbox, bool inbox) + { + var envelope = Create([0, 255, 128]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "first"u8.ToArray() } }; + var original = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + var repeated = new OutboxMessage + { + ReceiverId = change == "receiver" ? GrainId.Create("receiver", "other") : Receiver, + Envelope = envelope with + { + MessageId = change == "id" ? HierarchicalKey.Create("other") : change == "child" ? Id.CreateChildKey("child") : Id, + Payload = change == "payload" ? [0, 255, 129] : change == "payload-length" ? [0, 255] : [0, 255, 128], + Subject = change == "subject" ? "other" : change == "subject-case" ? "Subject.v1" : change == "no-subject" ? null : "subject.v1", + Headers = new() { ["custom"] = Encoding.UTF8.GetBytes(change == "headers" ? "second" : "first"), ["sender"] = change == "sender" ? [2] : [1] } + } + }; + Assert.Equal(outbox, Invoke("EnvelopeEquivalence", "AreEquivalent", original, repeated)); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.ToInboxMessage(), repeated.ToInboxMessage())); + Assert.Equal(inbox, Invoke("EnvelopeEquivalence", "AreSameCommand", original.Envelope, repeated.Envelope)); + Assert.Equal("first"u8.ToArray(), original.Envelope.Headers!["custom"]); + Assert.Equal(new byte[] { 0, 255, 128 }, original.Envelope.Payload); + } + + [Fact] + public void PendingRawCommandsAllowAbsentSubject() + { + var first = Create([]); + var repeated = first with { Headers = new() { ["custom"] = "different"u8.ToArray() } }; + Assert.True(Invoke("EnvelopeEquivalence", "AreSameCommand", first, repeated)); + Assert.Null(first.Subject); + Assert.Null(first.Headers); + } + + [Theory] + [InlineData("id", 1024, true)] + [InlineData("id", 1025, false)] + [InlineData("depth", 32, true)] + [InlineData("depth", 33, false)] + [InlineData("unicode-id", 512, true)] + [InlineData("unicode-id", 513, false)] + [InlineData("subject", 256, true)] + [InlineData("subject", 257, false)] + [InlineData("unicode-subject", 128, true)] + [InlineData("unicode-subject", 129, false)] + public void IdentityAndSubjectAdmissionBoundaries(string field, int length, bool valid) + { + var envelope = Create([]) with + { + MessageId = field switch + { + "id" => HierarchicalKey.Create(new string('x', length)), + "depth" => HierarchicalKey.Create(Enumerable.Repeat("x", length).ToArray()), + "unicode-id" => HierarchicalKey.Create(new string('\u00e9', length)), + _ => Id + }, + Subject = field.Contains("subject", StringComparison.Ordinal) ? new string(field == "subject" ? 's' : '\u00e9', length) : null + }; + if (valid) { Validate(envelope); Assert.Empty(envelope.Payload); } + else Assert.Throws(() => Validate(envelope)); + } + + [Theory] + [InlineData("empty-subject")] + [InlineData("invalid-subject")] + [InlineData("empty-header-key")] + [InlineData("null-header-value")] + public void InvalidMetadataIsRejectedBeforeAdmission(string invalid) + { + var envelope = Create([1]) with + { + Subject = invalid == "empty-subject" ? "" : invalid == "invalid-subject" ? "\ud800" : null, + Headers = invalid == "empty-header-key" ? new() { [""] = "value"u8.ToArray() } : invalid == "null-header-value" ? new() { ["key"] = null! } : null + }; + Assert.ThrowsAny(() => Validate(envelope)); + Assert.ThrowsAny(() => Validate(new InboxMessage { Envelope = envelope })); + Assert.ThrowsAny(() => Validate(new OutboxMessage { Envelope = envelope, ReceiverId = Receiver })); + Assert.Equal(new byte[] { 1 }, envelope.Payload); + } + + private static Envelope Create(byte[] payload) => new() { MessageId = Id, Payload = payload }; + private static void Validate(T message) => Invoke("EnvelopeValidation", "Validate", message!); + private static T Invoke(string type, string method, params object[] args) + { + var helper = typeof(Envelope).Assembly.GetType("Orleans.Messaging." + type, true)!; + try { return (T)helper.GetMethod(method, args.Select(a => a.GetType()).ToArray())!.Invoke(null, args)!; } + catch (TargetInvocationException exception) when (exception.InnerException is { } cause) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(cause).Throw(); + throw; + } + } + private static void AssertIds(Type type, params (string Name, uint Id)[] expected) + { + var actual = type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .SelectMany(member => member.GetCustomAttributes().Select(id => (member.Name, id.Id))) + .OrderBy(p => p.Id).ToArray(); + Assert.Equal(expected.OrderBy(p => p.Id), actual); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs new file mode 100644 index 00000000000..f81df0f7e25 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -0,0 +1,115 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class HandlerRoutingContractTests : IDisposable +{ + private readonly ServiceProvider _services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + + public void Dispose() => _services.Dispose(); + + [Fact] + public async Task Dispatch_SameSubjectRegisteredTwice_FirstRegistrationWins() + { + var type = CreateType("orders/created"); + var calls = new List(); + var dispatcher = new TestInbox() + .Register(type, (body, context) => { calls.Add($"first:{body}"); context.Complete(); }) + .Register(type, (body, context) => { calls.Add($"second:{body}"); context.Complete(); }); + var context = CreateContext(type, "a"); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal(["first:a"], calls); + Assert.Equal(1, context.Completions); + } + + [Fact] + public async Task Dispatch_ConsultsCanHandleInRegistrationOrder() + { + var order = new List(); + var dispatcher = new TestInbox(); + dispatcher.Register(new ProbeHandler("declines", canHandle: false, order)); + dispatcher.Register(new ProbeHandler("accepts", canHandle: true, order)); + dispatcher.Register(new ProbeHandler("unreached", canHandle: true, order)); + + await dispatcher.HandleAsync(CreateContext(CreateType("any"), "body"), CancellationToken.None); + + Assert.Equal(["declines?", "accepts?", "accepts!"], order); + } + + [Fact] + public async Task Dispatch_NoMatchingHandler_ThrowsWithSubject() + { + var dispatcher = new TestInbox().Register(CreateType("known"), (_, context) => context.Complete()); + var context = CreateContext(CreateType("unknown"), "body"); + + var exception = await Assert.ThrowsAsync( + () => dispatcher.HandleAsync(context, CancellationToken.None).AsTask()); + + Assert.Equal("No handler is registered for message subject 'unknown'.", exception.Message); + Assert.Equal(0, context.Completions); + } + + [Fact] + public void Dispatcher_IsEmptyUntilHandlerIsRegistered() + { + var dispatcher = new TestInbox(); + Assert.True(dispatcher.IsEmpty); + + dispatcher.Register(CreateType("subject"), (_, context) => context.Complete()); + + Assert.False(dispatcher.IsEmpty); + } + + [Fact] + public void HandlerNotFound_RoundTripsStatusAndDiagnostic() + { + var result = DeliveryResult.HandlerNotFound(); + var serializer = _services.GetRequiredService>(); + + var decoded = serializer.Deserialize(serializer.SerializeToArray(result)); + + Assert.Equal(DeliveryStatus.HandlerNotFound, decoded.Status); + Assert.Equal(result.Message, decoded.Message); + } + + private MessageType CreateType(string subject) => + new(subject, _services.GetRequiredService>()); + + private static FakeContext CreateContext(MessageType type, string body) => + new(new InboxMessage + { + Envelope = type.Create(HierarchicalKey.Create(Guid.NewGuid().ToString("N")), GrainId.Create("test", "receiver"), body).Envelope + }); + + private sealed class FakeContext(InboxMessage message) : IInboxHandlerContext + { + public int Completions { get; private set; } + public InboxMessage Message { get; } = message; + public void Complete() => Completions++; + public void Fail(string reason) => throw new NotSupportedException(); + } + + private sealed class ProbeHandler(string name, bool canHandle, List order) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) + { + order.Add($"{name}?"); + return canHandle; + } + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + order.Add($"{name}!"); + return ValueTask.CompletedTask; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs new file mode 100644 index 00000000000..de4ee9c5fa2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -0,0 +1,350 @@ +using System.Buffers; +using System.Reflection; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class HierarchicalKeyTests +{ + [Fact] + public void SerializationContract_IdentifiesCurrentType() + { + var type = typeof(HierarchicalKey); + Assert.Equal("Orleans.Messaging.HierarchicalKey", Assert.Single(type.GetCustomAttributes()).Alias); + Assert.True(type.IsValueType); + Assert.Single(type.GetCustomAttributes()); + Assert.Empty(type.GetCustomAttributes()); + var field = Assert.Single(type.GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.True(field.IsInitOnly); + Assert.True(field.FieldType.IsSealed); + Assert.Equal(typeof(string), field.FieldType.GetProperty("Canonical")!.PropertyType); + Assert.Equal(typeof(int), field.FieldType.GetProperty("Hash")!.PropertyType); + Assert.DoesNotContain(field.FieldType.GetFields(BindingFlags.Instance | BindingFlags.NonPublic), + value => value.FieldType == typeof(HierarchicalKey) || value.FieldType == field.FieldType); + } + + [Fact] + public void Default_IsDistinguishedUnsetValue() + { + var key = default(HierarchicalKey); + Assert.True(key.IsDefault); + Assert.Equal(0, key.Length); + Assert.Equal(0, key.CountSegments()); + Assert.Equal(0, key.GetHashCode()); + Assert.Equal(string.Empty, key.ToString()); + Assert.True(key == default); + Assert.False(key != default); + Assert.False(key.Equals(null)); + Assert.Null(key.GetParent()); + Assert.False(key.GetEnumerator().MoveNext()); + Assert.False(key.IsAncestorOf(default)); + Assert.False(key.IsParentOf(default)); + Assert.False(key.IsChildOf(default)); + Assert.False(HierarchicalKey.Create("a").IsAncestorOf(key)); + Assert.Throws(() => key.CreateChildKey("a")); + Assert.Throws(() => key.Append(HierarchicalKey.Create("a"))); + Assert.Throws(() => HierarchicalKey.Create("a").Append(default)); + Span empty = []; + Assert.True(key.TryFormat(empty, out var written, default, null)); + Assert.Equal(0, written); + } + + [Theory] + [InlineData("foo", "foo")] + [InlineData("foo/bar", @"foo\/bar")] + [InlineData(@"foo\bar", @"foo\\bar")] + [InlineData(@"foo\/bar/baz", @"foo\\\/bar\/baz")] + [InlineData("/", @"\/")] + [InlineData(@"\", @"\\")] + [InlineData("/a//b/", @"\/a\/\/b\/")] + [InlineData(" ", " ")] + public void Create_LiteralSegment_EscapesExactlyOnce(string literal, string canonical) + { + var key = HierarchicalKey.Create(literal); + Assert.Equal(canonical, key.ToString()); + Assert.Equal(canonical.Length, key.Length); + Assert.Equal(1, key.CountSegments()); + Assert.Null(key.GetParent()); + Assert.Equal(key, HierarchicalKey.Parse(canonical)); + Assert.Equal([canonical], Segments(key)); + Assert.Equal(key, HierarchicalKey.Create(new[] { literal })); + } + + [Fact] + public void Create_WithValues_CreatesRootFirstHierarchy() + { + var key = HierarchicalKey.Create("orders", "42", "payment"); + var root = HierarchicalKey.Create("orders"); + var parent = root.CreateChildKey("42"); + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(["orders", "42", "payment"], Segments(key)); + Assert.Equal(3, key.CountSegments()); + Assert.Equal(17, key.Length); + Assert.Equal(parent.CreateChildKey("payment"), key); + Assert.Equal(HierarchicalKey.Parse("orders/42/payment"), key); + Assert.Equal(parent, key.GetParent()); + Assert.Equal(root, key.GetParent()!.Value.GetParent()); + Assert.Null(key.GetParent()!.Value.GetParent()!.Value.GetParent()); + Assert.True(parent.IsParentOf(key)); + Assert.True(key.IsChildOf(parent)); + Assert.True(root.IsAncestorOf(key)); + Assert.False(root.IsParentOf(key)); + } + + [Fact] + public void Create_WithValues_ArrayAndSpanSlicesUseOnlyProvidedFragments() + { + string[] input = ["ignored", "orders", "42", "payment", "ignored"]; + var expected = HierarchicalKey.Parse("orders/42/payment"); + Assert.Equal(expected, HierarchicalKey.Create(input.AsSpan(1, 3))); + ReadOnlySpan slice = input.AsSpan(1, 3); + Assert.Equal(expected, HierarchicalKey.Create(slice)); + Assert.Equal(expected, HierarchicalKey.Create(["orders", "42", "payment"])); + } + + [Fact] + public void Create_WithValues_ArrayMutationDoesNotChangeKey() + { + string[] input = ["orders", "42", "payment"]; + var key = HierarchicalKey.Create(input); + var hash = key.GetHashCode(); + input[0] = "other"; + input[1] = null!; + Assert.Equal("orders/42/payment", key.ToString()); + Assert.Equal(hash, key.GetHashCode()); + Assert.Equal(HierarchicalKey.Parse("orders/42"), key.GetParent()); + } + + [Fact] + public void Create_WithValues_EmptyInputThrowsArgumentException() + { + Assert.Equal("values", Assert.Throws(() => HierarchicalKey.Create()).ParamName); + Assert.Throws(() => HierarchicalKey.Create(ReadOnlySpan.Empty)); + Assert.Throws(() => HierarchicalKey.Create(Array.Empty())); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_NullFragmentThrowsArgumentNullException(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = null!; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void Create_WithValues_EmptyLiteralSegmentRejects(int position) + { + string[] input = ["a", "b", "c"]; + input[position] = ""; + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(input)).ParamName); + } + + [Fact] + public void Create_LiteralSegment_RejectsOnlyNullAndEmpty() + { + Assert.Throws(() => HierarchicalKey.Create((string)null!)); + Assert.Throws(() => HierarchicalKey.Create("")); + var parent = HierarchicalKey.Create("root"); + Assert.Throws(() => parent.CreateChildKey(null!)); + Assert.Throws(() => parent.CreateChildKey("")); + } + + [Theory] + [InlineData("")] + [InlineData("/a")] + [InlineData("a/")] + [InlineData("a//b")] + [InlineData(@"a\")] + [InlineData(@"a\q")] + [InlineData(@"a\/b/")] + public void Parse_InvalidCanonicalPath_RejectsStringAndSpan(string path) + { + Assert.Throws(() => HierarchicalKey.Parse(path)); + Assert.Throws(() => HierarchicalKey.Parse(path.AsSpan())); + Assert.False(HierarchicalKey.TryParse(path, null, out var first)); + Assert.True(first.IsDefault); + Assert.False(HierarchicalKey.TryParse(path.AsSpan(), null, out var second)); + Assert.True(second.IsDefault); + } + + [Fact] + public void Parse_WithNullString_ThrowsArgumentNullException() + { + Assert.Throws(() => HierarchicalKey.Parse((string)null!)); + Assert.False(HierarchicalKey.TryParse(null, null, out var key)); + Assert.True(key.IsDefault); + } + + [Theory] + [InlineData("a/b/c", 3)] + [InlineData(@"a\/b/c", 2)] + [InlineData(@"a\\/b/c", 3)] + [InlineData(@"a\\\/b/c", 2)] + public void Parse_CanonicalEscapes_PreserveSegmentBoundaries(string path, int count) + { + var key = HierarchicalKey.Parse(path); + Assert.Equal(path, key.ToString()); + Assert.Equal(count, key.CountSegments()); + Assert.Equal(count, Segments(key).Count); + Assert.True(HierarchicalKey.TryParse(path.AsSpan(), null, out var copy)); + Assert.True(key == copy); + Assert.Equal(key.GetHashCode(), copy.GetHashCode()); + var characters = path.ToCharArray(); + var fromCaller = HierarchicalKey.Parse(characters.AsSpan()); + characters[0] = 'z'; + Assert.Equal(key, fromCaller); + } + + [Fact] + public void Append_ComposesConstructedPaths_ChildAppendsOneLiteral() + { + var parent = HierarchicalKey.Parse(@"root/fo\/o"); + var suffix = HierarchicalKey.Parse(@"ba\\r/baz"); + var composed = parent.Append(suffix); + Assert.Equal(@"root/fo\/o/ba\\r/baz", composed.ToString()); + Assert.Equal(4, composed.CountSegments()); + Assert.Equal(["root", @"fo\/o", @"ba\\r", "baz"], Segments(composed)); + Assert.Equal(parent.Append(HierarchicalKey.Create(@"ba\r")), composed.GetParent()); + Assert.True(parent.IsAncestorOf(composed)); + Assert.False(parent.IsParentOf(composed)); + var literal = parent.CreateChildKey("ba/r/baz"); + Assert.Equal(@"root/fo\/o/ba\/r\/baz", literal.ToString()); + Assert.Equal(3, literal.CountSegments()); + Assert.Equal(parent, literal.GetParent()); + Assert.True(parent.IsParentOf(literal)); + Assert.NotEqual(composed, literal); + } + + [Theory] + [InlineData("foo", "foo/bar", true, true)] + [InlineData("foo", "foo/bar/baz", false, true)] + [InlineData("foo/bar", "foo/bar", false, true)] + [InlineData("foo", "foobar/child", false, false)] + [InlineData("foo", @"foo\/bar/child", false, false)] + [InlineData("foo/bar", "foo/baz", false, false)] + [InlineData("Foo", "foo/bar", false, false)] + public void Navigation_UsesExactOrdinalSegments(string root, string path, bool immediate, bool ancestor) + { + var parent = HierarchicalKey.Parse(root); + var child = HierarchicalKey.Parse(path); + Assert.Equal(immediate, parent.IsParentOf(child)); + Assert.Equal(immediate, child.IsChildOf(parent)); + Assert.Equal(ancestor, parent.IsAncestorOf(child)); + if (immediate) Assert.Equal(parent, child.GetParent()); + } + + [Fact] + public void Equality_HashOperatorsAndDictionary_UseCanonicalOrdinalIdentity() + { + var direct = HierarchicalKey.Parse(@"tenant/acme/orders\/42"); + var composed = HierarchicalKey.Create("tenant", "acme", "orders/42"); + Assert.True(direct == composed); + Assert.False(direct != composed); + Assert.True(direct.Equals((object)composed)); + Assert.Equal(direct.GetHashCode(), composed.GetHashCode()); + Assert.Equal(direct.ToString().GetHashCode(StringComparison.Ordinal), direct.GetHashCode()); + var other = HierarchicalKey.Create("Tenant", "acme", "orders/42"); + Assert.True(other != direct); + Assert.False(other == direct); + Assert.False(direct.Equals("tenant/acme/orders\\/42")); + var dictionary = new Dictionary { [direct] = 42, [other] = 81, [default] = 5 }; + Assert.Equal(42, dictionary[composed]); + Assert.Equal(81, dictionary[other]); + Assert.Equal(5, dictionary[default]); + Assert.Equal(3, dictionary.Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SerializationAndCopy_RebuildCanonicalCacheAndPreserveValue(bool unset) + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var key = unset ? default : HierarchicalKey.Create("tenant", @"a\/b", "42"); + var serializer = first.GetRequiredService>(); + var bytes = serializer.SerializeToArray(key); + var decoded = second.GetRequiredService>().Deserialize(bytes); + var copy = first.GetRequiredService().Copy(key); + Assert.Equal(key, decoded); + Assert.Equal(key.IsDefault, decoded.IsDefault); + Assert.Equal(key.GetHashCode(), decoded.GetHashCode()); + Assert.Equal(key.CountSegments(), decoded.CountSegments()); + Assert.Equal(key, copy); + var data = Assert.Single(typeof(HierarchicalKey).GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); + Assert.Same(data.GetValue(key), data.GetValue(copy)); + if (!unset) Assert.NotSame(data.GetValue(key), data.GetValue(decoded)); + Assert.Equal(bytes, serializer.SerializeToArray(decoded)); + var map = first.GetRequiredService>>(); + var dictionary = Assert.IsType>(map.Deserialize(map.SerializeToArray(new() { [key] = 42 }))); + Assert.Equal(42, dictionary[decoded]); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Codec_InvalidCanonicalWire_RejectsInsteadOfConstructingKey(string canonical) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var session = services.GetRequiredService().GetSession(); + var output = new ArrayBufferWriter(); + var writer = Writer.Create(output, session); + ReferenceCodec.MarkValueField(session); + writer.WriteFieldHeader(0, typeof(HierarchicalKey), typeof(HierarchicalKey), WireType.TagDelimited); + services.GetRequiredService>().WriteField(ref writer, 0, typeof(string), canonical); + writer.WriteEndObject(); + writer.Commit(); + Assert.Throws(() => services.GetRequiredService>().Deserialize(output.WrittenSpan)); + } + + [Fact] + public void LongHierarchy_ConstructionNavigationAndFormatting_AreIterative() + { + var key = HierarchicalKey.Create(Enumerable.Repeat("x", 10000).ToArray()); + Assert.Equal(10000, key.CountSegments()); + Assert.Equal(19999, key.Length); + Assert.Equal(10000, Segments(key).Count); + Assert.Equal(9999, key.GetParent()!.Value.CountSegments()); + Assert.Equal(key, HierarchicalKey.Parse(key.ToString())); + Assert.True(HierarchicalKey.Create("x").IsAncestorOf(key)); + var span = new char[key.Length]; + Assert.True(key.TryFormat(span, out var written, default, null)); + Assert.Equal(key.Length, written); + Assert.Equal(key.ToString(), new string(span)); + Array.Fill(span, 'z'); + Assert.False(key.TryFormat(span.AsSpan(0, key.Length - 1), out written, default, null)); + Assert.Equal(0, written); + Assert.All(span, character => Assert.Equal('z', character)); + } + + [Fact] + public void EscapeCharacter_IsBackslash() => Assert.Equal('\\', HierarchicalKey.EscapeCharacter); + + [Fact] + public void SegmentSeparator_IsForwardSlash() => Assert.Equal('/', HierarchicalKey.SegmentSeparator); + + private static List Segments(HierarchicalKey key) + { + var values = new List(); + foreach (var segment in key) values.Add(segment.ToString()); + return values; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs new file mode 100644 index 00000000000..6995ba7bea4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs @@ -0,0 +1,2120 @@ +using System.Buffers; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Hosting; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class TypedMessageHelperTests +{ + private const string Subject = "orders.reserve"; + private static GrainId Sender => GrainId.Create("sender", "typed-helper"); + private static GrainId Receiver => GrainId.Create("receiver", "warehouse"); + private static HierarchicalKey MessageKey => HierarchicalKey.Create("orders", "42", "reserve"); + + [Fact] + public void MessageType_RejectsNullSerializer() + { + using var services = CreateServices(); + var exception = Assert.Throws( + () => new MessageType(Subject, null!)); + Assert.Equal("serializer", exception.ParamName); + } + + [Fact] + public void MessageType_RejectsNullSubject() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var exception = Assert.Throws( + () => new MessageType(null!, serializer)); + Assert.Equal("subject", exception.ParamName); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(Subject, new MessageType(Subject, serializer).Subject); + } + + [Theory] + [InlineData("empty")] + [InlineData("ascii")] + [InlineData("multibyte")] + public void MessageType_RejectsEmptyOrOversizedUtf8Subject(string variation) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var subject = InvalidSubject(variation); + + var exception = Assert.Throws(() => new MessageType(subject, serializer)); + Assert.Equal("subject", exception.ParamName); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(Subject, new MessageType(Subject, serializer).Subject); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void MessageType_AcceptsSubjectAtUtf8ByteLimit(bool multibyte) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var subject = SubjectAtLimit(multibyte); + var messageType = new MessageType(subject, serializer); + var envelope = DirectEnvelope(serializer, subject, "boundary reserve €42"); + + Assert.Equal(256, Encoding.UTF8.GetByteCount(subject)); + Assert.Equal(subject, messageType.Subject); + Assert.Equal("boundary reserve €42", messageType.Decode(envelope)); + } + + [Theory] + [InlineData("reserve order 42: € / 東京")] + [InlineData("")] + [InlineData("first line\nsecond line")] + public void MessageType_DecodeReadsDirectSerializerPayload(string body) + { + using var sendingServices = CreateServices(); + using var receivingServices = CreateServices(); + var sendingSerializer = sendingServices.GetRequiredService>(); + var receivingSerializer = receivingServices.GetRequiredService>(); + var envelope = DirectEnvelope(sendingSerializer, Subject, body); + var expectedWire = sendingSerializer.SerializeToArray(body); + var messageType = new MessageType(Subject, receivingSerializer); + + Assert.Equal(body, messageType.Decode(envelope)); + Assert.Equal(Subject, messageType.Subject); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Equal(body, receivingSerializer.Deserialize(envelope.Payload)); + } + + [Theory] + [InlineData("Orders.Reserve")] + [InlineData("orders.cancel")] + public void MessageType_DecodeRejectsWrongSubjectBeforeCodecRead(string wrongSubject) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + var sentinel = new InvalidDataException("read must not run"); + probe.ReadFailure = sentinel; + var messageType = new MessageType(Subject, serializer); + var envelope = DirectEnvelope(direct, wrongSubject, "reserve €42"); + var expectedWire = direct.SerializeToArray("reserve €42"); + + var exception = Assert.Throws(() => messageType.Decode(envelope)); + + Assert.Equal("envelope", exception.ParamName); + Assert.NotSame(sentinel, exception); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Equal("reserve €42", direct.Deserialize(envelope.Payload)); + } + + [Fact] + public void MessageType_DecodeRejectsDirectSerializedNull() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, null!); + var expectedWire = serializer.SerializeToArray(null); + var messageType = new MessageType(Subject, serializer); + Assert.Null(serializer.Deserialize(envelope.Payload)); + + var exception = Assert.Throws(() => messageType.Decode(envelope)); + + Assert.Equal("envelope", exception.ParamName); + Assert.Equal(Subject, messageType.Subject); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Null(serializer.Deserialize(envelope.Payload)); + } + + [Fact] + public void AddMessageType_RejectsNullServiceCollection() + { + var exception = Assert.Throws( + () => MessageTypeExtensions.AddMessageType(null!, Subject)); + Assert.Equal("services", exception.ParamName); + } + + [Fact] + public void AddMessageType_RejectsNullSubjectWithoutChangingServices() + { + var services = new ServiceCollection(); + services.AddSerializer(); + var before = services.ToArray(); + + var exception = Assert.Throws( + () => services.AddMessageType(null!)); + + Assert.Equal("subject", exception.ParamName); + AssertUnchangedServices(before, services); + } + + [Theory] + [InlineData("empty")] + [InlineData("ascii")] + [InlineData("multibyte")] + public void AddMessageType_RejectsEmptyOrOversizedUtf8SubjectWithoutChangingServices(string variation) + { + var services = new ServiceCollection(); + services.AddSerializer(); + var before = services.ToArray(); + var subject = InvalidSubject(variation); + + var exception = Assert.Throws(() => services.AddMessageType(subject)); + + Assert.Equal("subject", exception.ParamName); + AssertUnchangedServices(before, services); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessageType_AcceptsSubjectAtUtf8ByteLimit(bool multibyte) + { + var subject = SubjectAtLimit(multibyte); + var services = new ServiceCollection(); + services.AddSerializer(); + Assert.Same(services, services.AddMessageType(subject)); + using var provider = services.BuildServiceProvider(); + var descriptor = provider.GetRequiredKeyedService>(subject); + var envelope = DirectEnvelope(provider.GetRequiredService>(), subject, "registered boundary"); + + Assert.Equal(256, Encoding.UTF8.GetByteCount(subject)); + Assert.Equal(subject, descriptor.Subject); + Assert.Equal("registered boundary", descriptor.Decode(envelope)); + Assert.Same(descriptor, provider.GetRequiredKeyedService>(subject)); + } + + [Fact] + public void AddMessageType_ReturnsOriginalCollectionAndRegistersExactKeyedSingleton() + { + var services = new ServiceCollection(); + var result = services.AddMessageType(Subject); + Assert.Same(services, result); + Assert.Equal(3, services.Count); + var registration = Assert.Single(services, descriptor => descriptor.IsKeyedService); + Assert.True(registration.IsKeyedService); + Assert.Equal(Subject, registration.ServiceKey); + Assert.Equal(typeof(MessageType), registration.ServiceType); + Assert.Equal(ServiceLifetime.Singleton, registration.Lifetime); + + // Successful registration before adding serializers proves lazy resolution. + services.AddSerializer(); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var messageType = provider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, messageType.Subject); + Assert.Same(messageType, provider.GetRequiredKeyedService>(Subject)); + Assert.Same(messageType, scope.ServiceProvider.GetRequiredKeyedService>(Subject)); + var envelope = DirectEnvelope(provider.GetRequiredService>(), Subject, "keyed reserve"); + Assert.Equal("keyed reserve", messageType.Decode(envelope)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessageType_RegistersClosedSingletonSerializerFactoryBeforeKeyedBinding(bool valueType) + { + if (valueType) AssertClosedSerializerFactory(731); + else AssertClosedSerializerFactory("closed factory €42"); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void AddMessageType_PreservesExistingCustomClosedSerializerAndUsesItForBinding(bool valueType, bool factory) + { + if (valueType) AssertCustomClosedSerializerPreserved(731, factory); + else AssertCustomClosedSerializerPreserved("custom closed €42", factory); + } + + [Fact] + public void AddMessageType_AllowsMultipleSubjectsForSamePayloadType() + { + var services = new ServiceCollection(); + services.AddSerializer(); + services.AddMessageType(Subject).AddMessageType("orders.cancel"); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var reserve = provider.GetRequiredKeyedService>(Subject); + var cancel = provider.GetRequiredKeyedService>("orders.cancel"); + + Assert.Equal(Subject, reserve.Subject); + Assert.Equal("orders.cancel", cancel.Subject); + Assert.NotSame(reserve, cancel); + Assert.Same(reserve, provider.GetRequiredKeyedService>(Subject)); + Assert.Same(cancel, provider.GetRequiredKeyedService>("orders.cancel")); + Assert.Same(reserve, scope.ServiceProvider.GetRequiredKeyedService>(Subject)); + Assert.Same(cancel, scope.ServiceProvider.GetRequiredKeyedService>("orders.cancel")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessageType_RejectsSameAndCrossTypeDuplicateSubjectsWithoutChangingServices(bool crossType) + { + var services = new ServiceCollection(); + services.AddSerializer(); + services.AddMessageType(Subject); + var before = services.ToArray(); + + Assert.Throws(() => + { + if (crossType) + { + services.AddMessageType(Subject); + } + else + { + services.AddMessageType(Subject); + } + }); + + AssertUnchangedServices(before, services); + using var provider = services.BuildServiceProvider(); + var original = provider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, original.Subject); + var envelope = DirectEnvelope(provider.GetRequiredService>(), Subject, "original binding"); + Assert.Equal("original binding", original.Decode(envelope)); + Assert.Null(provider.GetKeyedService>(Subject)); + } + + [Fact] + public void AddMessageType_AllowsCaseDistinctSubjects() + { + var services = new ServiceCollection(); + services.AddSerializer(); + services.AddMessageType(Subject).AddMessageType("Orders.Reserve"); + using var provider = services.BuildServiceProvider(); + var lower = provider.GetRequiredKeyedService>(Subject); + var upper = provider.GetRequiredKeyedService>("Orders.Reserve"); + + Assert.Equal(Subject, lower.Subject); + Assert.Equal("Orders.Reserve", upper.Subject); + Assert.NotSame(lower, upper); + var lowerEnvelope = DirectEnvelope(provider.GetRequiredService>(), Subject, "lower"); + var upperEnvelope = DirectEnvelope(provider.GetRequiredService>(), "Orders.Reserve", "upper"); + Assert.Equal("lower", lower.Decode(lowerEnvelope)); + Assert.Equal("upper", upper.Decode(upperEnvelope)); + Assert.Throws(() => lower.Decode(upperEnvelope)); + } + + [Theory] + [InlineData("receiver")] + [InlineData("messageId")] + [InlineData("key-ascii-bytes")] + [InlineData("key-multibyte-bytes")] + public void MessageType_CreateRejectsInvalidIdentitiesBeforeCodecWrite(string invalidIdentity) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var receiver = invalidIdentity == "receiver" ? default : Receiver; + var key = invalidIdentity switch + { + "messageId" => default, + "key-ascii-bytes" => HierarchicalKey.Create(new string('k', 1025)), + "key-multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', 341) + "ab"), + _ => MessageKey + }; + OutboxMessage? result = null; + var exception = Assert.Throws(() => + { + result = messageType.Create(key, receiver, "must not encode"); + }); + Assert.Equal(invalidIdentity switch + { + "receiver" => "receiverId", + _ => "messageId" + }, exception.ParamName); + Assert.Null(result); + Assert.Equal(0, probe.WriteCount); + } + + [Fact] + public void MessageType_CreateUsesExplicitIdentitiesAndSuppliedHierarchicalKey() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var messageType = new MessageType(Subject, serializer); + var key = HierarchicalKey.Create("orders", "42", "reserve"); + var first = messageType.Create(key, Receiver, "reserve first"); + var otherReceiver = GrainId.Create("receiver", "other-warehouse"); + var second = messageType.Create(key, otherReceiver, "reserve second"); + + AssertEnvelopeIdentity(first, key, Receiver, Subject); + AssertEnvelopeIdentity(second, key, otherReceiver, Subject); + Assert.Equal("orders/42/reserve", first.Envelope.MessageId.ToString()); + Assert.Equal("orders/42/reserve", second.Envelope.MessageId.ToString()); + Assert.Equal(3, first.Envelope.MessageId.CountSegments()); + Assert.Equal(3, second.Envelope.MessageId.CountSegments()); + Assert.Equal("reserve first", serializer.Deserialize(first.Envelope.Payload)); + Assert.Equal("reserve second", serializer.Deserialize(second.Envelope.Payload)); + } + + [Theory] + [InlineData("segments")] + [InlineData("ascii-bytes")] + [InlineData("multibyte-bytes")] + public void MessageType_CreateAcceptsMessageKeyAtAdmissionLimits(string variation) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var key = variation switch + { + "segments" => HierarchicalKey.Create(Enumerable.Repeat("s", 32).ToArray()), + "ascii-bytes" => HierarchicalKey.Create(new string('k', 1024)), + "multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', 341) + "a"), + _ => throw new ArgumentOutOfRangeException(nameof(variation)) + }; + + var envelope = messageType.Create(key, Receiver, "admitted boundary"); + + Assert.Equal(1, probe.WriteCount); + AssertEnvelopeIdentity(envelope, key, Receiver, Subject); + if (variation == "segments") + { + Assert.Equal(32, envelope.Envelope.MessageId.CountSegments()); + } + else + { + Assert.Equal(1024, Encoding.UTF8.GetByteCount(envelope.Envelope.MessageId.ToString())); + } + Assert.Equal("admitted boundary", services.GetRequiredService>().Deserialize(envelope.Envelope.Payload)); + } + + [Theory] + [InlineData("string")] + [InlineData("empty-string")] + [InlineData("int")] + [InlineData("zero-int")] + [InlineData("negative-int")] + [InlineData("bytes")] + [InlineData("empty-bytes")] + public void MessageType_CreateMatchesDirectSerializerWireFormat(string contract) + { + using var services = CreateServices(); + using var receivingServices = CreateServices(); + switch (contract) + { + case "string": + AssertCreateWire(services, receivingServices, "reserve €42 / 東京"); + break; + case "empty-string": + AssertCreateWire(services, receivingServices, ""); + break; + case "int": + AssertCreateWire(services, receivingServices, 42_017); + break; + case "zero-int": + AssertCreateWire(services, receivingServices, 0); + break; + case "negative-int": + AssertCreateWire(services, receivingServices, -709); + break; + case "bytes": + AssertCreateWire(services, receivingServices, new byte[] { 0, 255, 128, 17, 42 }); + break; + case "empty-bytes": + AssertCreateWire(services, receivingServices, Array.Empty()); + break; + default: + throw new ArgumentOutOfRangeException(nameof(contract)); + } + + } + + [Fact] + public void MessageType_CreateAllocatesIndependentArraysAcrossSubsequentCalls() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var messageType = new MessageType(Subject, serializer); + var firstBody = new byte[] { 9, 8, 7, 255 }; + var largeBody = Enumerable.Range(0, 1_048_576).Select(i => (byte)(i * 31 + 7)).ToArray(); + var lastBody = new byte[] { 42, 0, 128, 255, 19, 21 }; + var largeWire = serializer.SerializeToArray(largeBody); + var lastWire = serializer.SerializeToArray(lastBody); + var first = messageType.Create(MessageKey, Receiver, firstBody); + var repeated = messageType.Create(MessageKey, Receiver, firstBody); + var large = messageType.Create(MessageKey, Receiver, largeBody); + var last = messageType.Create(MessageKey, Receiver, lastBody); + + Assert.NotNull(first.Envelope.Payload); + Assert.NotSame(firstBody, first.Envelope.Payload); + Assert.NotSame(first.Envelope.Payload, repeated.Envelope.Payload); + Assert.NotSame(first.Envelope.Payload, large.Envelope.Payload); + Assert.NotSame(large.Envelope.Payload, last.Envelope.Payload); + Assert.Equal(serializer.SerializeToArray(firstBody), first.Envelope.Payload.ToArray()); + Assert.Equal(first.Envelope.Payload.ToArray(), repeated.Envelope.Payload.ToArray()); + firstBody[0] = 99; + Assert.Equal(new byte[] { 9, 8, 7, 255 }, serializer.Deserialize(first.Envelope.Payload)); + Assert.Equal(new byte[] { 9, 8, 7, 255 }, serializer.Deserialize(repeated.Envelope.Payload)); + Assert.Equal(largeWire, large.Envelope.Payload.ToArray()); + Assert.Equal(lastWire, last.Envelope.Payload.ToArray()); + Assert.Equal(largeBody, serializer.Deserialize(large.Envelope.Payload)); + Assert.Equal(lastBody, serializer.Deserialize(last.Envelope.Payload)); + AssertEnvelopeIdentity(large, MessageKey, Receiver, Subject); + AssertEnvelopeIdentity(last, MessageKey, Receiver, Subject); + } + + [Fact] + public void MessageType_EncodingFailurePreservesOutstandingBytesAndPristineRetry() + { + using var services = CreateServices(); + var pristine = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var outstanding = messageType.Create(MessageKey, Receiver, "outstanding € order"); + var outstandingWire = pristine.SerializeToArray("outstanding € order"); + var sentinel = new InvalidDataException("committed partial codec failure"); + probe.NextWriteFailure = sentinel; + OutboxMessage? failedResult = null; + var exception = Assert.Throws(() => + { + failedResult = messageType.Create(MessageKey, Receiver, "failing preparation"); + }); + Assert.Same(sentinel, exception); + Assert.Null(failedResult); + Assert.Equal(2, probe.WriteCount); + Assert.Equal(1, probe.CommittedFailureCount); + Assert.Equal((byte)0x7e, probe.CommittedFailureByte); + + var recovered = messageType.Create(MessageKey, Receiver, "recovered 東京 reserve"); + Assert.Equal(2, probe.WriteCount); + Assert.Equal(pristine.SerializeToArray("recovered 東京 reserve"), recovered.Envelope.Payload.ToArray()); + Assert.Equal("recovered 東京 reserve", pristine.Deserialize(recovered.Envelope.Payload)); + Assert.Equal(outstandingWire, outstanding.Envelope.Payload.ToArray()); + Assert.Equal("outstanding € order", pristine.Deserialize(outstanding.Envelope.Payload)); + Assert.NotSame(outstanding.Envelope.Payload, recovered.Envelope.Payload); + AssertEnvelopeIdentity(recovered, MessageKey, Receiver, Subject); + } + + [Fact] + public void MessageType_EnvelopeCopySharesStableArrayAcrossSubsequentCreate() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var original = messageType.Create(MessageKey, Receiver, "surviving reserve"); + var surviving = original; + Assert.Same(original.Envelope.Payload, surviving.Envelope.Payload); + var expected = original.Envelope.Payload.ToArray(); + original = default; + var subsequent = messageType.Create(MessageKey.CreateChildKey("next"), Receiver, "new reserve"); + + Assert.Equal(2, probe.WriteCount); + Assert.NotSame(surviving.Envelope.Payload, subsequent.Envelope.Payload); + Assert.Equal(expected, surviving.Envelope.Payload.ToArray()); + Assert.Equal(MessageKey, surviving.Envelope.MessageId); + Assert.Equal(MessageKey.CreateChildKey("next"), subsequent.Envelope.MessageId); + Assert.Equal("surviving reserve", services.GetRequiredService>().Deserialize(surviving.Envelope.Payload)); + Assert.Equal("new reserve", services.GetRequiredService>().Deserialize(subsequent.Envelope.Payload)); + } + + [Fact] + public void MessageType_CreateRejectsNullBodyBeforeCodecWrite() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + + var exception = Assert.Throws( + () => messageType.Create(MessageKey, Receiver, null!)); + + Assert.Equal("body", exception.ParamName); + Assert.Equal(0, probe.WriteCount); + var control = messageType.Create(MessageKey, Receiver, "nonnull recovery"); + Assert.Equal(1, probe.WriteCount); + Assert.Equal("nonnull recovery", services.GetRequiredService>().Deserialize(control.Envelope.Payload)); + } + + [Fact] + public void MessageType_CreateLeavesResourceBodyAndDecodedValueApplicationOwned() + { + using var services = CreateServices(); + var codec = new ResourceCodec(); + var serializer = new Serializer(codec, services.GetRequiredService()); + var type = new MessageType(Subject, serializer); + using var body = new OwnedResource("resource €42"); + var original = type.Create(MessageKey, Receiver, body); + var retained = original; + Assert.Same(original.Envelope.Payload, retained.Envelope.Payload); + + Assert.Equal(0, body.DisposeCount); + Assert.Equal(MessageKey, retained.Envelope.MessageId); + Assert.Equal(services.GetRequiredService>().SerializeToArray(body.Name), retained.Envelope.Payload.ToArray()); + using var decoded = type.Decode(retained); + Assert.NotSame(body, decoded); + Assert.Same(codec.LastDecoded, decoded); + Assert.Equal("resource €42", decoded.Name); + Assert.Equal(0, decoded.DisposeCount); + Assert.Equal(1, codec.ReadCount); + } + + [Fact] + public void InboxDispatcher_RegisterRejectsNullMessageType() + { + var dispatcher = new TestInbox(); + var calls = 0; + var exception = Assert.Throws(() => dispatcher.Register(null!, (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + })); + + Assert.Equal("messageType", exception.ParamName); + Assert.Equal(0, calls); + } + + [Fact] + public void InboxDispatcher_RegisterRejectsNullHandler() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + var messageType = new MessageType(Subject, serializer); + var dispatcher = new TestInbox(); + + Func? handler = null; + var exception = Assert.Throws(() => dispatcher.Register(messageType, handler!)); + + Assert.Equal("handler", exception.ParamName); + Assert.Equal(0, probe.ReadCount); + dispatcher.Register(messageType, (_, _, _) => ValueTask.CompletedTask); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRoutesMultipleExactSubjectsAndPayloadTypes() + { + using var services = CreateServices(); + var strings = services.GetRequiredService>(); + var integers = services.GetRequiredService>(); + var dispatcher = new TestInbox(); + var lowerValues = new List(); + var upperValues = new List(); + var integerValues = new List(); + dispatcher.Register(new MessageType(Subject, strings), (body, _, _) => + { + lowerValues.Add(body); + return ValueTask.CompletedTask; + }); + dispatcher.Register(new MessageType("Orders.Reserve", strings), (body, _, _) => + { + upperValues.Add(body); + return ValueTask.CompletedTask; + }); + dispatcher.Register(new MessageType("orders.quantity", integers), (body, _, _) => + { + integerValues.Add(body); + return ValueTask.CompletedTask; + }); + var lowerEnvelope = DirectEnvelope(strings, Subject, "lower €42"); + var upperEnvelope = DirectEnvelope(strings, "Orders.Reserve", "upper 東京"); + var integerEnvelope = DirectEnvelope(integers, "orders.quantity", 42_017); + var lowerContext = new CountingInboxHandlerContext(lowerEnvelope); + var upperContext = new CountingInboxHandlerContext(upperEnvelope); + var integerContext = new CountingInboxHandlerContext(integerEnvelope); + + await dispatcher.HandleAsync(upperContext, CancellationToken.None); + Assert.Equal(new[] { "upper 東京" }, upperValues); + Assert.Empty(lowerValues); + Assert.Empty(integerValues); + await dispatcher.HandleAsync(integerContext, CancellationToken.None); + Assert.Equal(new[] { 42_017 }, integerValues); + Assert.Equal(new[] { "upper 東京" }, upperValues); + Assert.Empty(lowerValues); + await dispatcher.HandleAsync(lowerContext, CancellationToken.None); + Assert.Equal(new[] { "lower €42" }, lowerValues); + Assert.Equal(new[] { "upper 東京" }, upperValues); + Assert.Equal(new[] { 42_017 }, integerValues); + Assert.Equal(0, lowerContext.CompletionCount); + Assert.Equal(0, upperContext.CompletionCount); + Assert.Equal(0, integerContext.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRejectsUnknownSubjectBeforeDecodeOrHandler() + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("unknown subject must not decode"); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + }); + var direct = services.GetRequiredService>(); + var envelope = DirectEnvelope(direct, "orders.unknown", "valid reserve €42"); + var context = new CountingInboxHandlerContext(envelope); + + await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("valid reserve €42", direct.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRejectsMalformedKnownPayloadBeforeHandler() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var wire = serializer.SerializeToArray("nonempty truncated €42"); + var envelope = WireEnvelope(Subject, wire[..^1]); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + }); + // StringCodec.ReadRaw -> Reader.EnsureAvailable rejects the declared + // length exceeding remaining bytes, independently of subject routing. + Assert.Throws(() => serializer.Deserialize(envelope.Payload)); + + await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Equal(Subject, GetSubject(envelope)); + Assert.Equal(wire[..^1], envelope.Payload.ToArray()); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPassesExactContextAndCancellationToken() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "context €42"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var dispatcher = new TestInbox(); + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + CancellationToken capturedToken = default; + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, token) => + { + capturedBody = body; + capturedContext = caller; + capturedToken = token; + calls++; + return ValueTask.CompletedTask; + }); + + await dispatcher.HandleAsync(context, cancellation.Token); + + Assert.Equal("context €42", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(cancellation.Token, capturedToken); + Assert.True(capturedToken.CanBeCanceled); + Assert.False(capturedToken.IsCancellationRequested); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesSynchronousValueTaskCompletionWithoutAutomaticComplete() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "synchronous reserve"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + string? captured = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + return ValueTask.CompletedTask; + }); + + var dispatch = dispatcher.HandleAsync(context, CancellationToken.None); + Assert.True(dispatch.IsCompletedSuccessfully); + await dispatch; + + Assert.Equal("synchronous reserve", captured); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesSynchronousFaultedValueTask() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "faulted task reserve"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var sentinel = new InvalidDataException("synchronous faulted ValueTask"); + var calls = 0; + string? captured = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + return ValueTask.FromException(sentinel); + }); + + var dispatch = dispatcher.HandleAsync(context, CancellationToken.None); + Assert.True(dispatch.IsCompleted); + Assert.False(dispatch.IsCompletedSuccessfully); + var exception = await Assert.ThrowsAsync(async () => await dispatch); + + Assert.Same(sentinel, exception); + Assert.Equal("faulted task reserve", captured); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesSynchronousDelegateException() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "throwing delegate reserve"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var sentinel = new InvalidDataException("delegate throws before returning"); + var calls = 0; + string? captured = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + throw sentinel; + }); + + var exception = await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Same(sentinel, exception); + Assert.Equal("throwing delegate reserve", captured); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesDeferredValueTaskCompletion() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "deferred €42"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var dispatcher = new TestInbox(); + var calls = 0; + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + CancellationToken capturedToken = default; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, token) => + { + calls++; + capturedBody = body; + capturedContext = caller; + capturedToken = token; + return new ValueTask(completion.Task); + }); + + var dispatch = dispatcher.HandleAsync(context, cancellation.Token); + // Record the controlled incomplete state, then release and observe once + // before assertions, so even assertion failure cannot strand a dispatch. + var completedBeforeRelease = dispatch.IsCompleted; + var completionsBeforeRelease = context.CompletionCount; + var lateRegistration = Record.Exception(() => dispatcher.Register( + new MessageType("orders.later", serializer), (_, _, _) => ValueTask.CompletedTask)); + completion.SetResult(); + await dispatch; + + Assert.False(completedBeforeRelease); + Assert.IsType(lateRegistration); + Assert.Equal(0, completionsBeforeRelease); + Assert.Equal("deferred €42", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(cancellation.Token, capturedToken); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("deferred €42", serializer.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncPropagatesDeferredValueTaskFault() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "deferred fault €42"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sentinel = new InvalidDataException("controlled deferred fault"); + var dispatcher = new TestInbox(); + var calls = 0; + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + CancellationToken capturedToken = default; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, token) => + { + calls++; + capturedBody = body; + capturedContext = caller; + capturedToken = token; + return new ValueTask(completion.Task); + }); + + var dispatch = dispatcher.HandleAsync(context, cancellation.Token); + var completedBeforeRelease = dispatch.IsCompleted; + var completionsBeforeRelease = context.CompletionCount; + var lateRegistration = Record.Exception(() => dispatcher.Register( + new MessageType("orders.later", serializer), (_, _, _) => ValueTask.CompletedTask)); + completion.SetException(sentinel); + var exception = await Assert.ThrowsAsync(async () => await dispatch); + + Assert.False(completedBeforeRelease); + Assert.IsType(lateRegistration); + Assert.Equal(0, completionsBeforeRelease); + Assert.Same(sentinel, exception); + Assert.Equal("deferred fault €42", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(cancellation.Token, capturedToken); + Assert.Equal(1, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("deferred fault €42", serializer.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncStagesOnlyExplicitDelegateCompletion() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "explicit completion"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + string? capturedBody = null; + IInboxHandlerContext? capturedContext = null; + dispatcher.Register(new MessageType(Subject, serializer), (body, caller, _) => + { + calls++; + capturedBody = body; + capturedContext = caller; + caller.Complete(); + return ValueTask.CompletedTask; + }); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal("explicit completion", capturedBody); + Assert.Same(context, capturedContext); + Assert.Equal(1, calls); + Assert.Equal(1, context.CompletionCount); + Assert.Equal("explicit completion", serializer.Deserialize(envelope.Payload)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InboxDispatcher_HandleAsyncHonorsPreCanceledTokenWithoutDecodeHandlerOrComplete(bool unknownSubject) + { + using var services = CreateServices(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("canceled attempts must not decode"); + var direct = services.GetRequiredService>(); + var envelope = DirectEnvelope(direct, unknownSubject ? "orders.unknown" : Subject, "valid canceled reserve"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var calls = 0; + var dispatcher = new TestInbox(); + dispatcher.Register(new MessageType(Subject, serializer), (_, _, _) => + { + calls++; + return ValueTask.CompletedTask; + }); + + var exception = await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("valid canceled reserve", direct.Deserialize(envelope.Payload)); + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncLeavesDecodedResourceApplicationOwned() + { + using var services = CreateServices(); + var sessions = services.GetRequiredService(); + var codec = new ResourceCodec(); + var serializer = new Serializer(codec, sessions); + using var original = new OwnedResource("owned reserve €42"); + var envelope = DirectEnvelope(serializer, Subject, original); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + OwnedResource? captured = null; + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (body, _, _) => + { + calls++; + captured = body; + return ValueTask.CompletedTask; + }); + try + { + await dispatcher.HandleAsync(context, CancellationToken.None); + + var decoded = Assert.IsType(captured); + Assert.Same(codec.LastDecoded, decoded); + Assert.NotSame(original, decoded); + Assert.Equal("owned reserve €42", decoded.Name); + Assert.Equal(1, calls); + Assert.Equal(1, codec.ReadCount); + Assert.Equal(0, context.CompletionCount); + Assert.Equal(0, original.DisposeCount); + Assert.Equal(0, decoded.DisposeCount); + envelope = default; + Assert.Equal("owned reserve €42", decoded.Name); + Assert.Equal(0, decoded.DisposeCount); + decoded.Dispose(); + Assert.Equal(1, decoded.DisposeCount); + Assert.Equal(0, original.DisposeCount); + } + finally + { + if (codec.LastDecoded is { DisposeCount: 0 } resource) + { + resource.Dispose(); + } + } + } + + [Fact] + public async Task InboxDispatcher_HandleAsyncRejectsSerializedNullBeforeHandler() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, null!); + var expectedWire = serializer.SerializeToArray(null); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(new MessageType(Subject, serializer), (_, caller, _) => + { + calls++; + caller.Complete(); + return ValueTask.CompletedTask; + }); + Assert.Null(serializer.Deserialize(envelope.Payload)); + + var exception = await Assert.ThrowsAsync( + async () => await dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Equal("envelope", exception.ParamName); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal(expectedWire, envelope.Payload.ToArray()); + Assert.Null(serializer.Deserialize(envelope.Payload)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterAction_InvokesDecodedBodySynchronouslyWithExplicitCompletion(bool complete) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = DirectEnvelope(serializer, Subject, "synchronous"); + var context = new CountingInboxHandlerContext(envelope); + var dispatcher = new TestInbox(); + var calls = 0; + dispatcher.Register(type, (body, caller) => + { + Assert.Equal("synchronous", body); + Assert.Same(context, caller); + calls++; + if (complete) caller.Complete(); + }); + + var outcome = dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.True(outcome.IsCompletedSuccessfully); + Assert.Equal(1, calls); + Assert.Equal(complete ? 1 : 0, context.CompletionCount); + await outcome; + } + + [Fact] + public void RegisterAction_RejectsNullArguments() + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var dispatcher = new TestInbox(); + var missingType = Assert.Throws(() => + dispatcher.Register(null!, (_, _) => { })); + Action? handler = null; + var missingHandler = Assert.Throws(() => dispatcher.Register(type, handler!)); + + Assert.Equal("messageType", missingType.ParamName); + Assert.Equal("handler", missingHandler.ParamName); + dispatcher.Register(type, (_, _) => { }); + } + + [Fact] + public void RegisterAction_PropagatesOriginalDelegateFailure() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var envelope = DirectEnvelope(serializer, Subject, "failure"); + var context = new CountingInboxHandlerContext(envelope); + var sentinel = new IOException("synchronous handler failed"); + var dispatcher = new TestInbox().Register( + new MessageType(Subject, serializer), (_, _) => throw sentinel); + + var exception = Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Same(sentinel, exception); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterAction_CancellationBeforeEntryLeavesHandlerAndCompletionUntouched(bool cancelDuringDecode) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + using var cancellation = new CancellationTokenSource(); + if (cancelDuringDecode) probe.OnRead = cancellation.Cancel; + else cancellation.Cancel(); + var envelope = DirectEnvelope(direct, Subject, "cancellation"); + var context = new CountingInboxHandlerContext(envelope); + var calls = 0; + var dispatcher = new TestInbox().Register( + new MessageType(Subject, serializer), (_, _) => calls++); + + var exception = Assert.Throws(() => + dispatcher.HandleAsync(context, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(cancelDuringDecode ? 1 : 0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(0, context.CompletionCount); + } + + [Fact] + public async Task RegisterAction_DuplicateSubjectDispatchesToFirstRegistration() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var originalCalls = 0; + var replacementCalls = 0; + var dispatcher = new TestInbox().Register(type, (_, _) => originalCalls++); + dispatcher.Register(type, (_, _) => replacementCalls++); + var envelope = DirectEnvelope(serializer, Subject, "original"); + var context = new CountingInboxHandlerContext(envelope); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal(1, originalCalls); + Assert.Equal(0, replacementCalls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterStateAction_StaticDelegateReceivesExactReferenceStateBodyAndContext(bool complete) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "stateful €42"); + var context = new CountingInboxHandlerContext(envelope); + var state = new HandlerState { Complete = complete }; + Action handler = ObserveState; + Assert.Null(handler.Target); // Named static method, not a captured test closure. + var dispatcher = new TestInbox(); + dispatcher.Register(type, state, handler); + + var outcome = dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.True(outcome.IsCompletedSuccessfully); + await outcome; + Assert.Equal("stateful €42", state.Body); + Assert.Same(context, state.Context); + Assert.Same(state, state.Argument); + Assert.Equal(1, state.Calls); + Assert.Equal(complete ? 1 : 0, context.CompletionCount); + } + + [Fact] + public async Task RegisterState_MixedStaticReferenceAndStructArgumentsRouteOrdinalSubjects() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var syncType = new MessageType(Subject, serializer); + var asyncType = new MessageType("Orders.Reserve", serializer); + var syncState = new HandlerState { Complete = true }; + var asyncState = new HandlerState(); + var argument = new StructHandlerState(asyncState, 731); + Action sync = ObserveState; + Func asyncHandler = ObserveStructState; + Assert.Null(sync.Target); + Assert.Null(asyncHandler.Target); + var dispatcher = new TestInbox().Register(syncType, syncState, sync).Register(asyncType, argument, asyncHandler); + var syncEnvelope = syncType.Create(MessageKey, Receiver, "lower"); + var asyncEnvelope = asyncType.Create(MessageKey, Receiver, "upper"); + var syncContext = new CountingInboxHandlerContext(syncEnvelope); + var asyncContext = new CountingInboxHandlerContext(asyncEnvelope); + using var cancellation = new CancellationTokenSource(); + + await dispatcher.HandleAsync(syncContext, CancellationToken.None); + await dispatcher.HandleAsync(asyncContext, cancellation.Token); + + Assert.Equal("lower", syncState.Body); + Assert.Equal("upper", asyncState.Body); + Assert.Same(syncContext, syncState.Context); + Assert.Same(asyncContext, asyncState.Context); + Assert.Equal(731, asyncState.Marker); + Assert.Equal(cancellation.Token, asyncState.Token); + Assert.Equal(1, syncState.Calls); + Assert.Equal(1, asyncState.Calls); + Assert.Equal(1, syncContext.CompletionCount); + Assert.Equal(0, asyncContext.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_StaticStructArgumentIsStoredByValueAndInvokedExactly(bool asynchronous) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "struct state"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var observation = new HandlerState { Complete = true }; + var argument = new StructHandlerState(observation, 731); + var dispatcher = new TestInbox(); + if (asynchronous) + { + Func handler = ObserveStructState; + Assert.Null(handler.Target); + dispatcher.Register(type, argument, handler); + } + else + { + Action handler = ObserveStructAction; + Assert.Null(handler.Target); + dispatcher.Register(type, argument, handler); + } + argument = argument with { Marker = 999 }; + + await dispatcher.HandleAsync(context, cancellation.Token); + + Assert.Equal(999, argument.Marker); + Assert.Equal(731, observation.Marker); + Assert.Equal("struct state", observation.Body); + Assert.Same(context, observation.Context); + Assert.Same(observation, observation.Argument); + Assert.Equal(asynchronous ? cancellation.Token : default, observation.Token); + Assert.Equal(1, observation.Calls); + Assert.Equal(1, context.CompletionCount); + } + + [Theory] + [InlineData("success")] + [InlineData("fault")] + [InlineData("cancel")] + public async Task RegisterStateAsync_ReturnsActualDeferredValueTaskAndExactOutcome(string outcomeKind) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "deferred state"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var state = new HandlerState { Outcome = new ValueTask(gate.Task) }; + Func handler = ObserveAsyncState; + Assert.Null(handler.Target); + var dispatcher = new TestInbox().Register(type, state, handler); + + var outcome = dispatcher.HandleAsync(context, cancellation.Token); + + Assert.Equal(state.Outcome, outcome); // No async adapter replacing the actual ValueTask. + Assert.False(outcome.IsCompleted); + Assert.Equal("deferred state", state.Body); + Assert.Same(state, state.Argument); + Assert.Same(context, state.Context); + Assert.Equal(cancellation.Token, state.Token); + Assert.Equal(1, state.Calls); + Assert.Equal(0, context.CompletionCount); + var sentinel = new IOException("stateful deferred sentinel"); + if (outcomeKind == "fault") + { + gate.SetException(sentinel); + Assert.Same(sentinel, await Assert.ThrowsAsync(async () => await outcome)); + } + else if (outcomeKind == "cancel") + { + cancellation.Cancel(); + gate.SetCanceled(cancellation.Token); + var exception = await Assert.ThrowsAnyAsync(async () => await outcome); + Assert.Equal(cancellation.Token, exception.CancellationToken); + } + else + { + // Cancellation after a delegate's successful work must not change its outcome. + cancellation.Cancel(); + gate.SetResult(); + await outcome; + Assert.True(outcome.IsCompletedSuccessfully); + } + Assert.Equal(0, context.CompletionCount); + Assert.Equal(1, state.Calls); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_PropagatesOriginalSynchronousFailureWithoutCompleting(bool faultedValueTask) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "sentinel"); + var context = new CountingInboxHandlerContext(envelope); + var sentinel = new IOException("stateful sync sentinel"); + var state = new HandlerState(); + var dispatcher = new TestInbox(); + if (faultedValueTask) + { + state.Outcome = ValueTask.FromException(sentinel); + dispatcher.Register(type, state, ObserveAsyncState); + var outcome = dispatcher.HandleAsync(context, CancellationToken.None); + Assert.Equal(state.Outcome, outcome); + Assert.Same(sentinel, await Assert.ThrowsAsync(async () => await outcome)); + } + else + { + state.Failure = sentinel; + dispatcher.Register(type, state, ObserveState); + Assert.Same(sentinel, Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None))); + } + Assert.Equal("sentinel", state.Body); + Assert.Same(context, state.Context); + Assert.Equal(1, state.Calls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void RegisterState_CancellationBeforeOrDuringDecodePreventsAllDelegateEffects(bool asynchronous, bool duringDecode) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + using var cancellation = new CancellationTokenSource(); + if (duringDecode) probe.OnRead = cancellation.Cancel; + else cancellation.Cancel(); + var envelope = DirectEnvelope(direct, Subject, "canceled state"); + var context = new CountingInboxHandlerContext(envelope); + var state = new HandlerState { Complete = true }; + var type = new MessageType(Subject, serializer); + var dispatcher = new TestInbox(); + if (asynchronous) dispatcher.Register(type, state, ObserveAsyncState); + else dispatcher.Register(type, state, ObserveState); + + var exception = Assert.Throws(() => dispatcher.HandleAsync(context, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(duringDecode ? 1 : 0, probe.ReadCount); + Assert.Equal(0, state.Calls); + Assert.Null(state.Body); + Assert.Null(state.Context); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_CancellationInsideSuccessfulDelegatePreservesExplicitCompletion(bool asynchronous) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "completed despite cancel"); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + var state = new HandlerState { Complete = true, CancelOnInvoke = cancellation }; + var dispatcher = new TestInbox(); + if (asynchronous) dispatcher.Register(type, state, ObserveAsyncState); + else dispatcher.Register(type, state, ObserveState); + + var outcome = dispatcher.HandleAsync(context, cancellation.Token); + await outcome; + + Assert.True(cancellation.IsCancellationRequested); + Assert.True(outcome.IsCompletedSuccessfully); + Assert.Equal("completed despite cancel", state.Body); + Assert.Same(context, state.Context); + Assert.Equal(1, state.Calls); + Assert.Equal(1, context.CompletionCount); + } + + [Theory] + [InlineData(false, "unknown")] + [InlineData(true, "unknown")] + [InlineData(false, "malformed")] + [InlineData(true, "malformed")] + [InlineData(false, "codec-failure")] + [InlineData(true, "codec-failure")] + [InlineData(false, "null")] + [InlineData(true, "null")] + public void RegisterState_UnknownMalformedOrNullPayloadRejectsBeforeEffects(bool asynchronous, string variation) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + var sentinel = new InvalidDataException("malformed stateful payload"); + if (variation == "codec-failure") probe.ReadFailure = sentinel; + var wire = direct.SerializeToArray("nonempty truncated €42"); + var envelope = variation == "malformed" + ? WireEnvelope(Subject, wire[..^1]) + : DirectEnvelope(direct, variation == "unknown" ? "Orders.Reserve" : Subject, + variation == "null" ? null! : "payload"); + var context = new CountingInboxHandlerContext(envelope); + var type = new MessageType(Subject, serializer); + var state = new HandlerState { Complete = true }; + var dispatcher = new TestInbox(); + if (asynchronous) dispatcher.Register(type, state, ObserveAsyncState); + else dispatcher.Register(type, state, ObserveState); + + if (variation == "unknown") + Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None)); + else if (variation == "codec-failure") + Assert.Same(sentinel, Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None))); + else if (variation == "malformed") + { + Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None)); + Assert.Equal(wire[..^1], envelope.Payload.ToArray()); + } + else + Assert.Equal("envelope", Assert.Throws(() => + dispatcher.HandleAsync(context, CancellationToken.None)).ParamName); + + Assert.Equal(variation == "unknown" ? 0 : 1, probe.ReadCount); + Assert.Equal(0, state.Calls); + Assert.Null(state.Context); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RegisterState_DuplicateSameOrCrossTypeDispatchesToFirstRegistration(bool crossType) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var original = new HandlerState(); + var replacement = new HandlerState { Complete = true }; + var dispatcher = new TestInbox().Register(type, original, ObserveState); + if (crossType) + dispatcher.Register( + new MessageType(Subject, services.GetRequiredService>()), + replacement, ObserveNumberState); + else + dispatcher.Register(type, replacement, ObserveAsyncState); + var envelope = type.Create(MessageKey, Receiver, "original state"); + var context = new CountingInboxHandlerContext(envelope); + + await dispatcher.HandleAsync(context, CancellationToken.None); + + Assert.Equal("original state", original.Body); + Assert.Same(original, original.Argument); + Assert.Equal(1, original.Calls); + Assert.Equal(0, replacement.Calls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterState_RejectsNullTypeAndDelegateWithoutReservingSubject(bool asynchronous) + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var state = new HandlerState(); + var dispatcher = new TestInbox(); + if (asynchronous) + { + Assert.Equal("messageType", Assert.Throws(() => + dispatcher.Register(null!, state, ObserveAsyncState)).ParamName); + Assert.Equal("handler", Assert.Throws(() => dispatcher.Register(type, state, + (Func)null!)).ParamName); + dispatcher.Register(type, state, ObserveAsyncState); + } + else + { + Assert.Equal("messageType", Assert.Throws(() => + dispatcher.Register(null!, state, ObserveState)).ParamName); + Assert.Equal("handler", Assert.Throws(() => dispatcher.Register(type, state, + (Action)null!)).ParamName); + dispatcher.Register(type, state, ObserveState); + } + var envelope = type.Create(MessageKey, Receiver, "positive control"); + var context = new CountingInboxHandlerContext(envelope); + Assert.True(dispatcher.HandleAsync(context, CancellationToken.None).IsCompletedSuccessfully); + Assert.Equal("positive control", state.Body); + Assert.Equal(1, state.Calls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RegisterState_StructArgumentDispatchDoesNotAddPerInvocationBoxingAllocations(bool asynchronous) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var envelope = type.Create(MessageKey, Receiver, "allocation control"); + var context = new CountingInboxHandlerContext(envelope); + var baselineState = new HandlerState(); + var structState = new HandlerState(); + var baseline = new TestInbox(); + var withStruct = new TestInbox(); + if (asynchronous) + { + baseline.Register(type, baselineState, ObserveAsyncState); + withStruct.Register(type, new StructHandlerState(structState, 731), ObserveStructState); + } + else + { + baseline.Register(type, baselineState, ObserveState); + withStruct.Register(type, new StructHandlerState(structState, 731), ObserveStructAction); + } + // Same serializer/body/context, warmed routes and session pool. No assertions, + // async continuations, timing, or boxed arguments inside the measured loops. + MeasureDispatchAllocations(baseline, context, 64); + MeasureDispatchAllocations(withStruct, context, 64); + var referenceBytes = MeasureDispatchAllocations(baseline, context, 256); + var structBytes = MeasureDispatchAllocations(withStruct, context, 256); + + Assert.True(structBytes <= referenceBytes, $"Struct dispatch allocated {structBytes}, reference dispatch {referenceBytes}."); + Assert.Equal(320, baselineState.Calls); + Assert.Equal(320, structState.Calls); + Assert.Equal(731, structState.Marker); + Assert.Equal("allocation control", structState.Body); + Assert.Same(context, structState.Context); + Assert.Equal(0, context.CompletionCount); + } + + private static ServiceProvider CreateServices() => + new ServiceCollection().AddSerializer().BuildServiceProvider(); + + private static void AssertClosedSerializerFactory(T body) + { + var services = new ServiceCollection(); + services.AddMessageType(Subject).AddMessageType("orders.cancel"); + var closed = Assert.Single(services, descriptor => + !descriptor.IsKeyedService && descriptor.ServiceType == typeof(Serializer)); + Assert.Equal(ServiceLifetime.Singleton, closed.Lifetime); + Assert.Null(closed.ImplementationType); + Assert.Null(closed.ImplementationInstance); + var factory = closed.ImplementationFactory; + Assert.NotNull(factory); + var bindings = services.Where(descriptor => descriptor.IsKeyedService).ToArray(); + Assert.Equal(2, bindings.Length); + Assert.All(bindings, descriptor => Assert.True(services.IndexOf(closed) < services.IndexOf(descriptor))); + // Factory registration succeeds before serializer infrastructure exists: + // it must be lazy and must use the closed contract, including struct T. + services.AddSerializer(); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var serializer = provider.GetRequiredService>(); + Assert.Same(serializer, scope.ServiceProvider.GetRequiredService>()); + var factoryProvider = Substitute.For(); + factoryProvider.GetService(typeof(Serializer)).Returns(provider.GetRequiredService()); + factoryProvider.ClearReceivedCalls(); + var createdByFactory = Assert.IsType>(factory(factoryProvider)); + factoryProvider.Received(1).GetService(typeof(Serializer)); + factoryProvider.DidNotReceive().GetService(typeof(Serializer)); + Assert.Equal(serializer.SerializeToArray(body), createdByFactory.SerializeToArray(body)); + var first = provider.GetRequiredKeyedService>(Subject); + var second = provider.GetRequiredKeyedService>("orders.cancel"); + var envelope = first.Create(MessageKey, Receiver, body); + var other = second.Create(MessageKey, Receiver, body); + Assert.Equal(serializer.SerializeToArray(body), envelope.Envelope.Payload.ToArray()); + Assert.Equal(body, first.Decode(envelope)); + Assert.Equal(body, second.Decode(other)); + AssertEnvelopeIdentity(envelope, MessageKey, Receiver, Subject); + AssertEnvelopeIdentity(other, MessageKey, Receiver, "orders.cancel"); + Assert.NotSame(first, second); + } + + private static void AssertCustomClosedSerializerPreserved(T body, bool factory) + { + using var codecServices = CreateServices(); + var (custom, probe) = CreateProbedSerializer(codecServices); + var services = new ServiceCollection(); + var factoryCalls = 0; + if (factory) + services.AddSingleton>(_ => + { + factoryCalls++; + return custom; + }); + else services.AddSingleton(custom); + var original = Assert.Single(services); + + services.AddMessageType(Subject).AddMessageType("orders.cancel"); + + Assert.Same(original, Assert.Single(services, descriptor => + !descriptor.IsKeyedService && descriptor.ServiceType == typeof(Serializer))); + Assert.Equal(0, factoryCalls); + // No untyped Serializer service is installed here. Replacing this closed + // custom registration with the default factory would fail resolution. + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + Assert.Same(custom, provider.GetRequiredService>()); + Assert.Same(custom, scope.ServiceProvider.GetRequiredService>()); + var first = provider.GetRequiredKeyedService>(Subject); + var second = provider.GetRequiredKeyedService>("orders.cancel"); + var envelope = first.Create(MessageKey, Receiver, body); + var other = second.Create(MessageKey, Receiver, body); + Assert.Equal(2, probe.WriteCount); + Assert.Equal(codecServices.GetRequiredService>().SerializeToArray(body), envelope.Envelope.Payload.ToArray()); + Assert.Equal(body, first.Decode(envelope)); + Assert.Equal(body, second.Decode(other)); + Assert.Equal(2, probe.ReadCount); + Assert.Equal(factory ? 1 : 0, factoryCalls); + AssertEnvelopeIdentity(envelope, MessageKey, Receiver, Subject); + AssertEnvelopeIdentity(other, MessageKey, Receiver, "orders.cancel"); + } + + private static void ObserveState(string body, HandlerState argument, IInboxHandlerContext context) + { + argument.Calls++; + argument.Body = body; + argument.Context = context; + argument.Argument = argument; + if (argument.Failure is { } failure) throw failure; + argument.CancelOnInvoke?.Cancel(); + if (argument.Complete) context.Complete(); + } + + private static ValueTask ObserveAsyncState(string body, HandlerState argument, IInboxHandlerContext context, CancellationToken token) + { + argument.Token = token; + ObserveState(body, argument, context); + return argument.Outcome; + } + + private static ValueTask ObserveStructState(string body, StructHandlerState argument, IInboxHandlerContext context, CancellationToken token) + { + argument.Observation.Marker = argument.Marker; + return ObserveAsyncState(body, argument.Observation, context, token); + } + + private static void ObserveStructAction(string body, StructHandlerState argument, IInboxHandlerContext context) + { + argument.Observation.Marker = argument.Marker; + ObserveState(body, argument.Observation, context); + } + + private static ValueTask ObserveNumberState(int body, HandlerState argument, IInboxHandlerContext context, CancellationToken token) + { + argument.Marker = body; + return ObserveAsyncState("number", argument, context, token); + } + + private static long MeasureDispatchAllocations(IInbox dispatcher, IInboxHandlerContext context, int count) + { + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < count; i++) + dispatcher.HandleAsync(context, CancellationToken.None).GetAwaiter().GetResult(); + return GC.GetAllocatedBytesForCurrentThread() - before; + } + + private readonly record struct StructHandlerState(HandlerState Observation, int Marker); + + private sealed class HandlerState + { + public int Calls { get; set; } + public string? Body { get; set; } + public object? Argument { get; set; } + public IInboxHandlerContext? Context { get; set; } + public CancellationToken Token { get; set; } + public int Marker { get; set; } + public bool Complete { get; set; } + public CancellationTokenSource? CancelOnInvoke { get; set; } + public Exception? Failure { get; set; } + public ValueTask Outcome { get; set; } + } + + private static string InvalidSubject(string variation) => variation switch + { + "empty" => "", + "ascii" => new string('s', 257), + "multibyte" => new string('\u20ac', 85) + "ab", + _ => throw new ArgumentOutOfRangeException(nameof(variation)) + }; + + private static string SubjectAtLimit(bool multibyte) => + multibyte ? new string('\u20ac', 85) + "a" : new string('s', 256); + + private static (Serializer Serializer, ProbeCodec Probe) CreateProbedSerializer(IServiceProvider services) + { + var sessions = services.GetRequiredService(); + var probe = new ProbeCodec(sessions.CodecProvider.GetCodec()); + return (new Serializer(probe, sessions), probe); + } + + private static Envelope DirectEnvelope(Serializer serializer, string subject, [AllowNull] T body) => + Envelope(subject, serializer.SerializeToArray(body)); + + private static Envelope WireEnvelope(string subject, byte[] wire) => + Envelope(subject, wire); + + private static Envelope Envelope(string subject, byte[] payload) + { + return new Envelope + { + MessageId = MessageKey, + Payload = payload, + Subject = subject + }; + } + + private static void AssertEnvelopeIdentity( + OutboxMessage message, HierarchicalKey key, GrainId receiver, string subject) + { + Assert.Equal(key, message.Envelope.MessageId); + Assert.Equal(receiver, message.ReceiverId); + Assert.Equal(subject, GetSubject(message.Envelope)); + } + + private static string GetSubject(Envelope envelope) + { + return Assert.IsType(envelope.Subject); + } + + private static void AssertCreateWire( + IServiceProvider sendingServices, IServiceProvider receivingServices, T body) + { + var sending = sendingServices.GetRequiredService>(); + var receiving = receivingServices.GetRequiredService>(); + var messageType = new MessageType(Subject, sending); + var key = MessageKey; + var expectedWire = sending.SerializeToArray(body); + var envelope = messageType.Create(key, Receiver, body); + Assert.Equal(expectedWire, envelope.Envelope.Payload.ToArray()); + Assert.Equal(body, receiving.Deserialize(envelope.Envelope.Payload)); + AssertEnvelopeIdentity(envelope, key, Receiver, Subject); + } + + private static void AssertUnchangedServices(ServiceDescriptor[] before, IServiceCollection after) + { + Assert.Equal(before.Length, after.Count); + for (var i = 0; i < before.Length; i++) + { + Assert.Same(before[i], after[i]); + } + } + + private sealed class CountingInboxHandlerContext(InboxMessage message) : IInboxHandlerContext + { + public CountingInboxHandlerContext(Envelope envelope) : this(new InboxMessage { Envelope = envelope }) { } + public CountingInboxHandlerContext(OutboxMessage outgoing) : this(outgoing.ToInboxMessage()) { } + + public InboxMessage Message { get; } = message; + public int CompletionCount { get; private set; } + public void Complete() => CompletionCount++; + public void Fail(string reason) => throw new InvalidOperationException("Unexpected dead letter."); + } + + private sealed class ProbeCodec(IFieldCodec inner) : IFieldCodec + { + public int WriteCount { get; private set; } + public int ReadCount { get; private set; } + public int CommittedFailureCount { get; private set; } + public byte? CommittedFailureByte { get; private set; } + public Exception? NextWriteFailure { get; set; } + public Exception? ReadFailure { get; set; } + public Action? OnRead { get; set; } + + public void WriteField( + ref Writer writer, uint fieldIdDelta, [AllowNull] Type expectedType, [AllowNull] T value) + where TBufferWriter : IBufferWriter + { + WriteCount++; + if (NextWriteFailure is { } failure) + { + NextWriteFailure = null; + writer.WriteByte(0x7e); + writer.Commit(); + CommittedFailureByte = 0x7e; + CommittedFailureCount++; + throw failure; + } + + inner.WriteField(ref writer, fieldIdDelta, expectedType, value); + } + + [return: MaybeNull] + public T ReadValue(ref Reader reader, Field field) + { + ReadCount++; + if (ReadFailure is { } failure) throw failure; + var value = inner.ReadValue(ref reader, field); + OnRead?.Invoke(); + return value; + } + } + + private sealed class OwnedResource(string name) : IDisposable + { + public string Name { get; } = name; + public int DisposeCount { get; private set; } + public void Dispose() => DisposeCount++; + } + + private sealed class ResourceCodec : IFieldCodec + { + public OwnedResource? LastDecoded { get; private set; } + public int ReadCount { get; private set; } + + public void WriteField( + ref Writer writer, uint fieldIdDelta, [AllowNull] Type expectedType, + [AllowNull] OwnedResource value) where TBufferWriter : IBufferWriter + { + ArgumentNullException.ThrowIfNull(value); + // Manual local wire contract: the resource's name uses genuine string + // encoding. No global registration or generated disposal/copy behavior. + StringCodec.WriteField(ref writer, fieldIdDelta, value.Name); + } + + [return: MaybeNull] + public OwnedResource ReadValue(ref Reader reader, Field field) + { + ReadCount++; + var name = StringCodec.ReadValue(ref reader, field) + ?? throw new InvalidDataException("Resource name must not be null."); + return LastDecoded = new OwnedResource(name); + } + } + + [Fact] + public void MessageType_DecodeRejectsAbsentSubjectBeforeCodecRead() + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("payload codec must not run"); + var type = new MessageType(Subject, serializer); + var envelope = new Envelope { MessageId = MessageKey, Payload = direct.SerializeToArray("unrouted €42") }; + var outgoing = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; + + foreach (var decode in new Func[] + { + () => type.Decode(envelope), + () => type.Decode(outgoing), + () => type.Decode(outgoing.ToInboxMessage()) + }) + { + Assert.Equal("envelope", Assert.Throws(() => decode()).ParamName); + } + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, probe.WriteCount); + Assert.Equal("unrouted €42", direct.Deserialize(envelope.Payload)); + Assert.Equal(MessageKey, envelope.MessageId); + Assert.Null(envelope.Subject); + Assert.Null(envelope.Headers); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void InboxDispatcher_SubjectHeaderFailurePrecedesCodecHandlerAndOutboxMutation( + bool absent, bool asynchronous) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var (serializer, probe) = CreateProbedSerializer(services); + probe.ReadFailure = new InvalidDataException("codec must not run"); + var type = new MessageType(Subject, serializer); + var envelope = new Envelope + { + MessageId = MessageKey, + Payload = direct.SerializeToArray("untouched"), + Subject = absent ? null : "Orders.Reserve" + }; + var context = new CountingInboxHandlerContext(envelope); + var outbox = Substitute.For(); + var stock = 7; + var calls = 0; + var dispatcher = new TestInbox(); + void Mutate(string body, IInboxHandlerContext caller) + { + calls++; + stock--; + outbox.Send(type, MessageKey.CreateChildKey("result"), Receiver, body); + caller.Complete(); + } + if (asynchronous) + dispatcher.Register(type, (body, caller, _) => + { + Mutate(body, caller); + return ValueTask.CompletedTask; + }); + else dispatcher.Register(type, Mutate); + + var exception = Assert.Throws(() => + dispatcher.HandleAsync(context, CancellationToken.None)); + + Assert.Contains(absent ? "requires a subject" : "No handler is registered", exception.Message, + StringComparison.Ordinal); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, calls); + Assert.Equal(7, stock); + Assert.Equal(0, context.CompletionCount); + outbox.DidNotReceive().Send(Arg.Any()); + Assert.Equal("untouched", direct.Deserialize(context.Message.Envelope.Payload)); + Assert.Equal(MessageKey, context.Message.Envelope.MessageId); + if (absent) + { + Action validate = EnvelopeValidation.Validate; + foreach (var invalidSubject in new[] { "", new string('s', 257), new string('\u20ac', 85) + "ab" }) + { + var invalid = new Envelope + { + MessageId = MessageKey, + Payload = envelope.Payload, + Subject = invalidSubject + }; + Assert.Equal("subject", Assert.Throws(() => + validate(invalid)).ParamName); + Assert.Same(envelope.Payload, invalid.Payload); + } + var invalidUtf16 = new Envelope + { + MessageId = MessageKey, + Payload = envelope.Payload, + Subject = "\ud800" + }; + Assert.Throws(() => validate(invalidUtf16)); + var missingPayload = new Envelope { MessageId = MessageKey, Payload = null!, Subject = Subject }; + Assert.Equal("envelope", Assert.Throws(() => + validate(missingPayload)).ParamName); + var valid = new Envelope { MessageId = MessageKey, Payload = envelope.Payload, Subject = Subject }; + validate(valid); + Assert.Equal(Subject, GetSubject(valid)); + Assert.Equal("untouched", direct.Deserialize(valid.Payload)); + Assert.Equal(0, probe.ReadCount); + Assert.Equal(0, calls); + Assert.Equal(7, stock); + Assert.Equal(0, context.CompletionCount); + outbox.DidNotReceive().Send(Arg.Any()); + } + } + + [Fact] + public void MessageType_CreateSharesEnvelopeObjectsAndDecodesAllMessageRoles() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + + var outgoing = type.Create(MessageKey, Receiver, "shared payload € / 東京"); + var incoming = outgoing.ToInboxMessage(); + var envelope = outgoing.Envelope; + + Assert.Equal(MessageKey, envelope.MessageId); + Assert.Equal(Receiver, outgoing.ReceiverId); + Assert.Equal(Subject, GetSubject(envelope)); + Assert.Null(envelope.Headers); + Assert.Equal(serializer.SerializeToArray("shared payload € / 東京"), envelope.Payload); + Assert.Same(envelope.Payload, incoming.Envelope.Payload); + Assert.Same(envelope.Subject, incoming.Envelope.Subject); + Assert.Null(incoming.Envelope.Headers); + Assert.Equal("shared payload € / 東京", type.Decode(envelope)); + Assert.Equal("shared payload € / 東京", type.Decode(outgoing)); + Assert.Equal("shared payload € / 東京", type.Decode(incoming)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Envelope_IdAndPayloadOnlyPermitEmptyPayloadAndOptionalMetadata(bool emptyPayload) + { + byte[] bytes = emptyPayload ? [] : [0, 17, 255]; + var envelope = new Envelope { MessageId = MessageKey, Payload = bytes }; + + Assert.Equal(MessageKey, envelope.MessageId); + Assert.Equal(bytes, envelope.Payload.ToArray()); + Assert.Same(bytes, envelope.Payload); + Assert.Null(envelope.Subject); + Assert.Null(envelope.Headers); + } + + [Fact] + public void InboxDispatcher_SubjectHeaderLookupDoesNotAllocateDecodedSubjectStrings() + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(SubjectAtLimit(multibyte: true), serializer); + var outgoing = type.Create(MessageKey, Receiver, "allocation control"); + var context = new CountingInboxHandlerContext(outgoing); + var state = new HandlerState(); + var dispatcher = new TestInbox().Register(type, state, ObserveState); + for (var i = 0; i < 64; i++) type.Decode(outgoing); + MeasureDispatchAllocations(dispatcher, context, 64); + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 256; i++) type.Decode(outgoing); + var decodeBytes = GC.GetAllocatedBytesForCurrentThread() - before; + + var dispatchBytes = MeasureDispatchAllocations(dispatcher, context, 256); + + Assert.True(dispatchBytes <= decodeBytes, + $"Subject dispatch allocated {dispatchBytes}, direct decode allocated {decodeBytes}."); + Assert.Equal(320, state.Calls); + Assert.Equal("allocation control", state.Body); + Assert.Same(context, state.Context); + Assert.Equal(0, context.CompletionCount); + Assert.Equal(SubjectAtLimit(multibyte: true), GetSubject(outgoing.Envelope)); + Assert.Equal(MessageKey, outgoing.Envelope.MessageId); + Assert.Equal("allocation control", type.Decode(context.Message)); + } + + [Fact] + public void AddMessageType_GrainIdPayloadCollapsesSharedSerializerRegistration() + { + var services = new ServiceCollection(); + + Assert.Same(services, services.AddMessageType(Subject)); + + Assert.Equal(2, services.Count); + var closed = services[0]; + Assert.False(closed.IsKeyedService); + Assert.Equal(typeof(Serializer), closed.ServiceType); + Assert.Equal(ServiceLifetime.Singleton, closed.Lifetime); + Assert.Null(closed.ImplementationInstance); + Assert.Null(closed.ImplementationType); + Assert.NotNull(closed.ImplementationFactory); + var binding = services[1]; + Assert.True(binding.IsKeyedService); + Assert.Equal(typeof(MessageType), binding.ServiceType); + Assert.Equal(Subject, binding.ServiceKey); + Assert.Equal(ServiceLifetime.Singleton, binding.Lifetime); + var bindingFactory = binding.KeyedImplementationFactory; + Assert.NotNull(bindingFactory); + + // Registration must be lazy: the serializer infrastructure is added later. + services.AddSerializer(); + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + var serializer = provider.GetRequiredService>(); + var messageType = provider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, messageType.Subject); + Assert.Same(serializer, scope.ServiceProvider.GetRequiredService>()); + Assert.Same(messageType, scope.ServiceProvider.GetRequiredKeyedService>(Subject)); + Assert.Same(messageType, provider.GetRequiredKeyedService>(Subject)); + var factoryProvider = Substitute.For(); + factoryProvider.GetService(typeof(Serializer)).Returns(serializer); + factoryProvider.ClearReceivedCalls(); + + var fromFactory = Assert.IsType>(bindingFactory(factoryProvider, Subject)); + + Assert.Equal(Subject, fromFactory.Subject); + factoryProvider.Received(1).GetService(typeof(Serializer)); + factoryProvider.DidNotReceive().GetService(typeof(Serializer)); + + // An existing closed serializer serves both roles without replacement or + // an untyped Serializer dependency, and its factory runs only once. + using var codecServices = CreateServices(); + var (custom, probe) = CreateProbedSerializer(codecServices); + var customServices = new ServiceCollection(); + var factoryCalls = 0; + customServices.AddSingleton>(_ => + { + factoryCalls++; + return custom; + }); + var original = Assert.Single(customServices); + + Assert.Same(customServices, customServices.AddMessageType(Subject)); + + Assert.Equal(2, customServices.Count); + Assert.Same(original, customServices[0]); + Assert.Same(original, Assert.Single(customServices, descriptor => + !descriptor.IsKeyedService && descriptor.ServiceType == typeof(Serializer))); + Assert.Equal(0, factoryCalls); + using var customProvider = customServices.BuildServiceProvider(); + using var customScope = customProvider.CreateScope(); + Assert.Null(customProvider.GetService()); + var customBinding = customProvider.GetRequiredKeyedService>(Subject); + Assert.Equal(Subject, customBinding.Subject); + Assert.Same(customBinding, customScope.ServiceProvider.GetRequiredKeyedService>(Subject)); + Assert.Same(custom, customProvider.GetRequiredService>()); + Assert.Same(custom, customScope.ServiceProvider.GetRequiredService>()); + Assert.Equal(1, factoryCalls); + Assert.Equal(0, probe.WriteCount); + Assert.Equal(0, probe.ReadCount); + + var bytes = custom.SerializeToArray(Receiver); + + Assert.Equal(codecServices.GetRequiredService>().SerializeToArray(Receiver), bytes); + Assert.Equal(Receiver, custom.Deserialize(bytes)); + Assert.Equal(1, probe.WriteCount); + Assert.Equal(1, probe.ReadCount); + Assert.Equal(1, factoryCalls); + + var outgoing = customBinding.Create(MessageKey, Receiver, Receiver); + Assert.Equal(2, probe.WriteCount); + Assert.Equal(Receiver, outgoing.ReceiverId); + Assert.Equal(MessageKey, outgoing.Envelope.MessageId); + Assert.Equal(Receiver, customBinding.Decode(outgoing)); + Assert.Null(outgoing.Envelope.Headers); + Assert.Equal(2, probe.ReadCount); + Assert.Equal(1, factoryCalls); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs new file mode 100644 index 00000000000..95adc5fd79d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs @@ -0,0 +1,600 @@ +using System.Buffers; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class TypedOutboxExtensionsTests +{ + private const string Subject = "orders.result"; + private static GrainId Owner => GrainId.Create("warehouse", "reply-owner"); + private static GrainId ReceivedSender => GrainId.Create("customer", "original-command-sender"); + private static GrainId Destination => GrainId.Create("audit", "explicit-destination"); + private static HierarchicalKey CommandId => HierarchicalKey.Create("orders", "42", "reserve"); + + [Fact] + public void SendBatch_EnumeratesOnceAndStagesEachMessageBeforeAdvancing() + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var outbox = new StoringOutbox(); + var enumerations = 0; + IEnumerable<(HierarchicalKey, GrainId, string)> Messages() + { + Assert.Equal(1, ++enumerations); + yield return (CommandId, Destination, "first"); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal("first", type.Decode(Assert.Single(outbox.Messages))); + yield return (CommandId.CreateChildKey("second"), ReceivedSender, "second"); + } + + outbox.Send(type, Messages()); + + Assert.Equal(1, enumerations); + Assert.Equal(2, outbox.RawSendCalls); + Assert.Equal(new[] { CommandId, CommandId.CreateChildKey("second") }, outbox.Messages.Select(value => value.Envelope.MessageId)); + Assert.Equal(new[] { Destination, ReceivedSender }, outbox.Messages.Select(value => value.ReceiverId)); + Assert.All(outbox.Messages, value => Assert.Equal(Subject, GetSubject(value.Envelope))); + Assert.Equal(new[] { "first", "second" }, outbox.Messages.Select(type.Decode)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SendBatch_PreparationFailurePreservesEarlierIntentAndOriginalError(bool encodingFailure) + { + using var services = CreateServices(); + var sessions = services.GetRequiredService(); + var codec = new FailingCodec(sessions.CodecProvider.GetCodec()); + var type = new MessageType(Subject, new Serializer(codec, sessions)); + var outbox = new StoringOutbox(); + var sentinel = new InvalidDataException("later preparation failed"); + IEnumerable<(HierarchicalKey, GrainId, string)> Messages() + { + yield return (CommandId, Destination, "first"); + if (!encodingFailure) throw sentinel; + codec.NextFailure = sentinel; + yield return (CommandId.CreateChildKey("second"), Destination, "second"); + } + + Assert.Same(sentinel, Assert.Throws(() => outbox.Send(type, Messages()))); + Assert.Equal(1, outbox.RawSendCalls); + var earlier = Assert.Single(outbox.Messages); + Assert.Equal(CommandId, earlier.Envelope.MessageId); + Assert.Equal(Destination, earlier.ReceiverId); + Assert.Equal(Subject, GetSubject(earlier.Envelope)); + var earlierPayload = earlier.Envelope.Payload.ToArray(); + Assert.Equal("first", type.Decode(earlier)); + Assert.Equal(encodingFailure ? 2 : 1, codec.Writes); + outbox.Send(type, CommandId.CreateChildKey("second"), Destination, "recovered"); + Assert.Equal(new[] { "first", "recovered" }, outbox.Messages.Select(type.Decode)); + Assert.Equal(2, outbox.RawSendCalls); + var retained = outbox.Messages.First(); + Assert.Equal(earlierPayload, retained.Envelope.Payload); + Assert.Equal(new[] { CommandId, CommandId.CreateChildKey("second") }, + outbox.Messages.Select(message => message.Envelope.MessageId)); + Assert.All(outbox.Messages, message => Assert.Equal(Destination, message.ReceiverId)); + } + + [Theory] + [InlineData("outbox")] + [InlineData("messageType")] + [InlineData("messages")] + public void SendBatch_RejectsNullArgumentsBeforeEncodingOrStaging(string variation) + { + using var services = CreateServices(); + var actual = new StoringOutbox(); + var type = new MessageType(Subject, services.GetRequiredService>()); + IOutbox outbox = actual; + IEnumerable<(HierarchicalKey, GrainId, string)> messages = [(CommandId, Destination, "body")]; + switch (variation) + { + case "outbox": outbox = null!; break; + case "messageType": type = null!; break; + case "messages": messages = null!; break; + } + + Assert.Equal(variation, Assert.Throws(() => outbox.Send(type, messages)).ParamName); + Assert.Equal(0, actual.RawSendCalls); + Assert.Empty(actual.Messages); + } + + [Fact] + public void SendBatch_EmptyBatchStagesNothing() + { + using var services = CreateServices(); + var outbox = new StoringOutbox(); + var type = new MessageType(Subject, services.GetRequiredService>()); + + outbox.Send(type, []); + + Assert.Equal(0, outbox.RawSendCalls); + Assert.Empty(outbox.Messages); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SendBatch_RawFailurePreservesPublishedArrayAndOriginalError(bool storeBeforeFailure) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var body = new byte[3_145_728]; + body[body.Length / 2] = 17; + var sentinel = new IOException("raw staging failed"); + var outbox = new StoringOutbox() { Failure = sentinel, StoreBeforeFailure = storeBeforeFailure }; + (HierarchicalKey, GrainId, byte[])[] messages = + [ + (CommandId, Destination, body), + (CommandId.CreateChildKey("second"), Destination, body) + ]; + + Assert.Same(sentinel, Assert.Throws(() => outbox.Send(type, messages))); + Assert.Equal(1, outbox.RawSendCalls); + var published = outbox.LastPublished!.Value; + Assert.NotNull(outbox.WireSeen); + Assert.Same(published.Envelope.Payload, outbox.WireSeen); + Assert.Equal(serializer.SerializeToArray(body), published.Envelope.Payload.ToArray()); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + if (storeBeforeFailure) + { + Assert.Same(published.Envelope.Payload, Assert.Single(outbox.Messages).Envelope.Payload); + outbox.ClearMessages(); + } + Assert.Empty(outbox.Messages); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Send_SerializesExactEnvelopeWithExplicitDestination(bool addressable) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var outbox = new StoringOutbox(); + IAddressable reference = new AddressableDestination(Destination); + + if (addressable) outbox.Send(type, CommandId, reference, "typed reserve € / 東京"); + else outbox.Send(type, CommandId, Destination, "typed reserve € / 東京"); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(1, outbox.Count); + Assert.Equal(CommandId, staged.Envelope.MessageId); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal(Subject, GetSubject(staged.Envelope)); + Assert.Equal(serializer.SerializeToArray("typed reserve € / 東京"), staged.Envelope.Payload.ToArray()); + Assert.Equal("typed reserve € / 東京", serializer.Deserialize(staged.Envelope.Payload)); + AssertEnvelopeFields(staged.Envelope); + Assert.True(outbox.TryGetMessage(CommandId, out var found)); + Assert.Same(staged.Envelope.Payload, found.Envelope.Payload); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SendReply_UsesResultChildAndExplicitDestination(bool addressable) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var received = new MessageType("orders.reserve", serializer) + .Create(CommandId, Owner, "incoming reserve"); + var context = new CountingContext(received.ToInboxMessage()); + var outbox = new StoringOutbox(); + IAddressable reference = new AddressableDestination(Destination); + + if (addressable) outbox.SendReply(type, context, reference, "approved €42"); + else outbox.SendReply(type, context, Destination, "approved €42"); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(CommandId.CreateChildKey("result"), staged.Envelope.MessageId); + Assert.Equal("orders/42/reserve/result", staged.Envelope.MessageId.ToString()); + Assert.Equal(4, staged.Envelope.MessageId.CountSegments()); + Assert.Equal(Destination, staged.ReceiverId); + Assert.NotEqual(ReceivedSender, staged.ReceiverId); + Assert.Equal(Subject, GetSubject(staged.Envelope)); + Assert.Equal(serializer.SerializeToArray("approved €42"), staged.Envelope.Payload.ToArray()); + Assert.Equal("approved €42", serializer.Deserialize(staged.Envelope.Payload)); + AssertEnvelopeFields(staged.Envelope); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + Assert.Equal("incoming reserve", serializer.Deserialize(received.Envelope.Payload)); + Assert.Equal(CommandId, received.Envelope.MessageId); + } + + [Theory] + [InlineData(false, "segments")] + [InlineData(true, "segments")] + [InlineData(false, "ascii-bytes")] + [InlineData(true, "ascii-bytes")] + [InlineData(false, "multibyte-bytes")] + [InlineData(true, "multibyte-bytes")] + public void SendReply_ResultChildAtAdmissionLimitEncodesAndStages(bool addressable, string variation) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var key = variation switch + { + "segments" => HierarchicalKey.Create(Enumerable.Repeat("s", 31).ToArray()), + "ascii-bytes" => HierarchicalKey.Create(new string('k', 1017)), + "multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', 339)), + _ => throw new ArgumentOutOfRangeException(nameof(variation)) + }; + var context = Context(key); + var outbox = new StoringOutbox(); + + Invoke(outbox, type, context, key, Destination, "boundary result", reply: true, addressable: addressable); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(key.CreateChildKey("result"), staged.Envelope.MessageId); + if (variation == "segments") Assert.Equal(32, staged.Envelope.MessageId.CountSegments()); + else Assert.Equal(1024, System.Text.Encoding.UTF8.GetByteCount(staged.Envelope.MessageId.ToString())); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal(serializer.SerializeToArray("boundary result"), staged.Envelope.Payload.ToArray()); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void TypedSend_StoresPublishedArrayAndPreservesBytesAfterRemoval(bool reply, bool addressable) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var body = Enumerable.Range(0, 3_145_728).Select(i => (byte)(i * 31 + 7)).ToArray(); + var wire = serializer.SerializeToArray(body); + var outbox = new StoringOutbox(); + var context = Context(); + + Invoke(outbox, type, context, CommandId, Destination, body, reply, addressable); + + var staged = Assert.Single(outbox.Messages); + var published = outbox.LastPublished!.Value; + Assert.NotNull(outbox.WireSeen); + Assert.Equal(wire, staged.Envelope.Payload.ToArray()); + Assert.Equal(body, serializer.Deserialize(staged.Envelope.Payload)); + Assert.Same(published.Envelope.Payload, staged.Envelope.Payload); + Assert.Same(published.Envelope.Payload, outbox.WireSeen); + Assert.NotSame(body, published.Envelope.Payload); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + Assert.Equal(reply ? CommandId.CreateChildKey("result") : CommandId, staged.Envelope.MessageId); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal(0, context.CompletionCount); + outbox.ClearMessages(); + + Assert.Empty(outbox.Messages); + Assert.Equal(0, outbox.Count); + Assert.Equal(wire, published.Envelope.Payload.ToArray()); + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + Assert.Equal(1, outbox.RawSendCalls); + Invoke(outbox, type, context, CommandId, Destination, body, reply, addressable); + var subsequent = Assert.Single(outbox.Messages); + Assert.NotSame(published.Envelope.Payload, subsequent.Envelope.Payload); + Assert.Equal(wire, subsequent.Envelope.Payload.ToArray()); + Assert.Equal(wire, published.Envelope.Payload.ToArray()); + Assert.Equal(2, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, true, false)] + [InlineData(true, false, false)] + [InlineData(true, true, false)] + [InlineData(false, false, true)] + [InlineData(false, true, true)] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + public void TypedSend_RawSendFailurePreservesExactExceptionAndPublishedArray(bool reply, bool addressable, bool storeBeforeFailure) + { + using var services = CreateServices(); + var serializer = services.GetRequiredService>(); + var type = new MessageType(Subject, serializer); + var body = Enumerable.Range(0, 3_145_728).Select(i => (byte)(i * 17 + 3)).ToArray(); + var sentinel = new IOException("rawSend sentinel"); + var outbox = new StoringOutbox() { Failure = sentinel, StoreBeforeFailure = storeBeforeFailure }; + var context = Context(); + + var exception = Assert.Throws(() => + Invoke(outbox, type, context, CommandId, Destination, body, reply, addressable)); + + Assert.Same(sentinel, exception); + Assert.Equal(1, outbox.RawSendCalls); + Assert.NotNull(outbox.WireSeen); + Assert.Equal(serializer.SerializeToArray(body), outbox.WireSeen.ToArray()); + var published = outbox.LastPublished!.Value; + Assert.Equal(Destination, published.ReceiverId); + Assert.Equal(reply ? CommandId.CreateChildKey("result") : CommandId, published.Envelope.MessageId); + Assert.Equal(Subject, GetSubject(published.Envelope)); + Assert.Same(outbox.WireSeen, published.Envelope.Payload); + if (storeBeforeFailure) + { + var stored = Assert.Single(outbox.Messages); + Assert.Same(published.Envelope.Payload, stored.Envelope.Payload); + Assert.Equal(body, serializer.Deserialize(stored.Envelope.Payload)); + outbox.ClearMessages(); + } + Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); + Assert.Empty(outbox.Messages); + Assert.Equal(0, outbox.Count); + Assert.Equal(0, context.CompletionCount); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void TypedSend_PartialEncodingFailureNeverStagesAndNextCallHasPristineWire(bool reply, bool addressable) + { + using var services = CreateServices(); + var direct = services.GetRequiredService>(); + var sessions = services.GetRequiredService(); + var codec = new FailingCodec(sessions.CodecProvider.GetCodec()); + var type = new MessageType(Subject, new Serializer(codec, sessions)); + var outbox = new StoringOutbox(); + var context = Context(); + var sentinel = new InvalidDataException("committed encoder sentinel"); + codec.NextFailure = sentinel; + + var exception = Assert.Throws(() => + Invoke(outbox, type, context, CommandId, Destination, "failing", reply, addressable)); + + Assert.Same(sentinel, exception); + Assert.Equal(1, codec.Writes); + Assert.Equal(1, codec.CommittedFailures); + Assert.Equal(0, outbox.RawSendCalls); + Assert.Null(outbox.LastPublished); + Assert.Null(outbox.WireSeen); + Assert.Empty(outbox.Messages); + Assert.Equal(0, context.CompletionCount); + Invoke(outbox, type, context, CommandId, Destination, "recovered €42", reply, addressable); + var staged = Assert.Single(outbox.Messages); + Assert.Equal(direct.SerializeToArray("recovered €42"), staged.Envelope.Payload.ToArray()); + Assert.Equal("recovered €42", direct.Deserialize(staged.Envelope.Payload)); + Assert.Equal(reply ? CommandId.CreateChildKey("result") : CommandId, staged.Envelope.MessageId); + Assert.Equal(2, codec.Writes); + Assert.Equal(1, outbox.RawSendCalls); + Assert.Equal(0, context.CompletionCount); + } + + public static IEnumerable ValidationCases() + { + foreach (var reply in new[] { false, true }) + foreach (var addressable in new[] { false, true }) + { + foreach (var variation in new[] { "outbox", "messageType", "body", "bytes", "multibyte-bytes" }) + yield return [reply, addressable, variation]; + if (reply) + { + yield return [reply, addressable, "context"]; + yield return [reply, addressable, "unset-parent"]; + } + else yield return [reply, addressable, "messageId"]; + yield return [reply, addressable, addressable ? "null-reference" : "destination"]; + if (addressable) yield return [reply, addressable, "invalid-reference"]; + } + } + + [Theory] + [MemberData(nameof(ValidationCases))] + public void TypedSend_ValidationFailureNeverEncodesOrStages(bool reply, bool addressable, string variation) + { + using var services = CreateServices(); + var sessions = services.GetRequiredService(); + var codec = new FailingCodec(sessions.CodecProvider.GetCodec()); + var type = new MessageType(Subject, new Serializer(codec, sessions)); + var realOutbox = new StoringOutbox(); + IOutbox outbox = variation == "outbox" ? null! : realOutbox; + var key = variation switch + { + "messageId" or "unset-parent" => default, + // Reply adds seven UTF8 bytes ("/result"). + "bytes" => HierarchicalKey.Create(new string('k', reply ? 1018 : 1025)), + "multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', reply ? 339 : 341) + (reply ? "a" : "ab")), + _ => CommandId + }; + var context = Context(reply ? key : null); + var destination = variation is "destination" or "invalid-reference" ? default : Destination; + var body = variation == "body" ? null! : "never encoded"; + if (variation == "messageType") type = null!; + if (variation == "context") context = null!; + + var exception = Record.Exception(() => + { + if (addressable) + { + IAddressable reference = variation == "null-reference" ? null! : new AddressableDestination(destination); + if (reply) outbox.SendReply(type, context, reference, body); + else outbox.Send(type, key, reference, body); + } + else if (reply) outbox.SendReply(type, context, destination, body); + else outbox.Send(type, key, destination, body); + }); + + if (variation == "unset-parent") + { + Assert.IsType(exception); + } + else + { + var argumentError = Assert.IsAssignableFrom(exception); + Assert.Equal(variation switch + { + "null-reference" => "destination", + "invalid-reference" => "grain", + "destination" => "receiverId", + "bytes" or "multibyte-bytes" or "messageId" => "messageId", + _ => variation + }, argumentError.ParamName); + } + Assert.Equal(0, codec.Writes); + Assert.Equal(0, realOutbox.RawSendCalls); + Assert.Null(realOutbox.LastPublished); + Assert.Null(realOutbox.WireSeen); + Assert.Empty(realOutbox.Messages); + Assert.Equal(0, realOutbox.Count); + if (context is not null) Assert.Equal(0, context.CompletionCount); + // Positive control: same codec still encodes and stages after rejection. + var positive = new StoringOutbox(); + Invoke(positive, + new MessageType(Subject, new Serializer(codec, sessions)), + Context(), CommandId, Destination, "valid control", reply, addressable); + Assert.Equal(1, codec.Writes); + Assert.Equal(1, positive.RawSendCalls); + var staged = Assert.Single(positive.Messages); + Assert.Equal(services.GetRequiredService>().SerializeToArray("valid control"), staged.Envelope.Payload.ToArray()); + Assert.Equal(Destination, staged.ReceiverId); + } + + private static ServiceProvider CreateServices() => + new ServiceCollection().AddSerializer().BuildServiceProvider(); + + private static CountingContext Context(HierarchicalKey? key = null) + { + var messageId = key ?? CommandId; + if (messageId == default) return new(new InboxMessage { Envelope = default }); + using var services = CreateServices(); + return new(new InboxMessage + { + Envelope = new Envelope + { + MessageId = messageId, + Payload = [], + Subject = "orders.reserve", + Headers = new Dictionary(StringComparer.Ordinal) + { + ["sender"] = services.GetRequiredService>().SerializeToArray(ReceivedSender) + } + } + }); + } + + private static void Invoke(IOutbox outbox, MessageType type, CountingContext context, + HierarchicalKey key, GrainId destination, T body, bool reply, bool addressable) + { + if (addressable) + { + IAddressable reference = new AddressableDestination(destination); + if (reply) outbox.SendReply(type, context, reference, body); + else outbox.Send(type, key, reference, body); + } + else if (reply) outbox.SendReply(type, context, destination, body); + else outbox.Send(type, key, destination, body); + } + + private sealed class AddressableDestination : IAddressable, IGrainBase + { + public AddressableDestination(GrainId id) + { + GrainContext = Substitute.For(); + GrainContext.GrainId.Returns(id); + } + public IGrainContext GrainContext { get; } + } + + private sealed class CountingContext(InboxMessage message) : IInboxHandlerContext + { + public InboxMessage Message { get; } = message; + public int CompletionCount { get; private set; } + public void Complete() => CompletionCount++; + public void Fail(string reason) => throw new InvalidOperationException("Unexpected dead letter."); + } + + private sealed class StoringOutbox : IOutbox + { + private readonly List _messages = []; + public int Count => _messages.Count; + public IEnumerable Messages => _messages; + public int RawSendCalls { get; private set; } + public OutboxMessage? LastPublished { get; private set; } + public byte[]? WireSeen { get; private set; } + public Exception? Failure { get; set; } + public bool StoreBeforeFailure { get; set; } + + public void Send(OutboxMessage message) + { + RawSendCalls++; + LastPublished = message; + WireSeen = message.Envelope.Payload; + if (Failure is { } failure) + { + if (StoreBeforeFailure) _messages.Add(message); + throw failure; + } + _messages.Add(message); + } + + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) + { + foreach (var message in _messages) + if (message.Envelope.MessageId == messageId) + { + envelope = message; + return true; + } + envelope = default; + return false; + } + + public void ClearMessages() => _messages.Clear(); + } + + private static string GetSubject(Envelope envelope) + { + return Assert.IsType(envelope.Subject); + } + + + private static void AssertEnvelopeFields(Envelope envelope) + { + Assert.Equal(Subject, envelope.Subject); + Assert.Null(envelope.Headers); + } + + private sealed class FailingCodec(IFieldCodec inner) : IFieldCodec + { + public int Writes { get; private set; } + public int CommittedFailures { get; private set; } + public Exception? NextFailure { get; set; } + + public void WriteField(ref Writer writer, uint fieldIdDelta, + [AllowNull] Type expectedType, [AllowNull] string value) where TBufferWriter : IBufferWriter + { + Writes++; + if (NextFailure is { } failure) + { + NextFailure = null; + writer.WriteByte(0x7e); + writer.Commit(); + CommittedFailures++; + throw failure; + } + inner.WriteField(ref writer, fieldIdDelta, expectedType, value); + } + + [return: MaybeNull] + public string ReadValue(ref Reader reader, Field field) => inner.ReadValue(ref reader, field); + } +} diff --git a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj new file mode 100644 index 00000000000..5b9a87425a4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj @@ -0,0 +1,19 @@ + + + $(TestTargetFrameworks) + true + enable + enable + $(NoWarn);ORLEANSEXP005 + + + + + + + + + + + + diff --git a/test/Orleans.Messaging.Tests/Support/HierarchicalKeyTestExtensions.cs b/test/Orleans.Messaging.Tests/Support/HierarchicalKeyTestExtensions.cs new file mode 100644 index 00000000000..b1a0fd1b284 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/HierarchicalKeyTestExtensions.cs @@ -0,0 +1,15 @@ +namespace Orleans.Messaging.Tests.Support; + +internal static class HierarchicalKeyTestExtensions +{ + public static int CountSegments(this HierarchicalKey key) + { + var count = 0; + foreach (var _ in key) + { + count++; + } + + return count; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/TestInbox.cs b/test/Orleans.Messaging.Tests/Support/TestInbox.cs new file mode 100644 index 00000000000..d5808df5f55 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/TestInbox.cs @@ -0,0 +1,24 @@ +namespace Orleans.Messaging.Tests.Support; + +/// An which dispatches directly to its registered handlers. +internal sealed class TestInbox : IInbox +{ + private readonly InboxDispatcher _dispatcher = new([]); + + public bool IsEmpty => _dispatcher.IsEmpty; + + public IInbox Register(IInboxHandler handler) + { + _dispatcher.Add(handler); + return this; + } + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => + _dispatcher.HandleAsync(context, cancellationToken); +} + +internal static class TestInboxExtensions +{ + public static ValueTask HandleAsync(this IInbox inbox, IInboxHandlerContext context, CancellationToken cancellationToken) => + ((TestInbox)inbox).HandleAsync(context, cancellationToken); +} From 869dfaf8e02be62cf67072c0ad35eb0f754d7509 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 14:23:20 -0700 Subject: [PATCH 02/95] refactor(messaging): rename InboxOptions to DurableMessagingOptions The options configure both the inbox and the outbox. Also syncs the layer-1 contract tests with #10693: message ID depth is bounded only by encoded size, HierarchicalKey holds a single init-only string field, and handler registration remains open during dispatch. --- ...xOptions.cs => DurableMessagingOptions.cs} | 8 ++--- src/Orleans.Messaging/README.md | 4 +-- .../DeliveryAndOptionsContractTests.cs | 32 +++++++++---------- .../Contracts/EnvelopeContractTests.cs | 5 +-- .../Contracts/HierarchicalKeyTests.cs | 6 +--- .../Contracts/TypedMessageHelperTests.cs | 14 +++----- 6 files changed, 30 insertions(+), 39 deletions(-) rename src/Orleans.Messaging/Configuration/{InboxOptions.cs => DurableMessagingOptions.cs} (97%) diff --git a/src/Orleans.Messaging/Configuration/InboxOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs similarity index 97% rename from src/Orleans.Messaging/Configuration/InboxOptions.cs rename to src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index f0a6bfaf4ca..82c7d025f72 100644 --- a/src/Orleans.Messaging/Configuration/InboxOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -3,12 +3,12 @@ namespace Orleans.Messaging.Configuration; /// -/// Configuration options for the inbox messaging system. +/// Configuration options for durable messaging inboxes, outboxes, and the messaging pump. /// /// /// -/// These options control the behavior of the inbox, including capacity limits, -/// deduplication tracking, retry behavior, and pump batch sizes. +/// These options control inbox capacity, deduplication tracking, inbox and outbox retry and +/// dead-letter behavior, and pump batch sizes. /// /// /// Transport is at-least-once. Deduplication provides effectively-once handler effects only @@ -16,7 +16,7 @@ namespace Orleans.Messaging.Configuration; /// throughput, and recovery characteristics. /// /// -public class InboxOptions +public class DurableMessagingOptions { internal const int MaximumBackoffExponent = 6; internal const int MaximumBackoffMultiplier = 1 << MaximumBackoffExponent; diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 3e34b3285d4..9fd3a2022b7 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -73,5 +73,5 @@ Each receiver returns a `DeliveryResult`: - `HandlerNotFound`: the receiver has no registered handlers. - `DeadLettered`: the message moved to the receiver's dead letters. -`InboxOptions` configures capacity, batch sizes, retries, deduplication, and -dead-letter retention. +`DurableMessagingOptions` configures capacity, batch sizes, retries, deduplication, +and dead-letter retention. diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index 16cd4a72d6d..0cf2d33aefe 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -41,7 +41,7 @@ public void DeliveryStatus_AllValues_HaveContiguousDistinctValues() [Fact] public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() { - var options = new InboxOptions(); + var options = new DurableMessagingOptions(); options.Validate(); @@ -59,24 +59,24 @@ public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() [Fact] public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContract() { - var invalidCases = new (string Parameter, Action Mutate)[] + var invalidCases = new (string Parameter, Action Mutate)[] { - (nameof(InboxOptions.MaxCapacity), options => options.MaxCapacity = 0), - (nameof(InboxOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), - (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), - (nameof(InboxOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), - (nameof(InboxOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), - (nameof(InboxOptions.MaxDeliveryAttempts), options => options.MaxDeliveryAttempts = 0), - (nameof(InboxOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), - (nameof(InboxOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), - (nameof(InboxOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), - (nameof(InboxOptions.InboxBatchSize), options => options.InboxBatchSize = 0), - (nameof(InboxOptions.OutboxBatchSize), options => options.OutboxBatchSize = 0), + (nameof(DurableMessagingOptions.MaxCapacity), options => options.MaxCapacity = 0), + (nameof(DurableMessagingOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), + (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), + (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), + (nameof(DurableMessagingOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), + (nameof(DurableMessagingOptions.MaxDeliveryAttempts), options => options.MaxDeliveryAttempts = 0), + (nameof(DurableMessagingOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), + (nameof(DurableMessagingOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), + (nameof(DurableMessagingOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), + (nameof(DurableMessagingOptions.InboxBatchSize), options => options.InboxBatchSize = 0), + (nameof(DurableMessagingOptions.OutboxBatchSize), options => options.OutboxBatchSize = 0), }; foreach (var (parameter, mutate) in invalidCases) { - var options = new InboxOptions(); + var options = new DurableMessagingOptions(); mutate(options); var exception = Assert.Throws(options.Validate); Assert.Equal(parameter, exception.ParamName); @@ -89,14 +89,14 @@ public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoun { foreach (var retryAge in new[] { TimeSpan.FromHours(2), TimeSpan.FromHours(3) }) { - var options = new InboxOptions + var options = new DurableMessagingOptions { DeduplicationWindow = TimeSpan.FromHours(2), MaxOutboxRetryAge = retryAge, }; var exception = Assert.Throws(options.Validate); - Assert.Equal(nameof(InboxOptions.MaxOutboxRetryAge), exception.ParamName); + Assert.Equal(nameof(DurableMessagingOptions.MaxOutboxRetryAge), exception.ParamName); Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); } } diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index cb017e464db..9e06c6f86f9 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -250,8 +250,9 @@ public void PendingRawCommandsAllowAbsentSubject() [Theory] [InlineData("id", 1024, true)] [InlineData("id", 1025, false)] - [InlineData("depth", 32, true)] - [InlineData("depth", 33, false)] + // Segment depth is bounded only by the encoded message ID size. + [InlineData("depth", 512, true)] + [InlineData("depth", 513, false)] [InlineData("unicode-id", 512, true)] [InlineData("unicode-id", 513, false)] [InlineData("subject", 256, true)] diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs index de4ee9c5fa2..298dd156c77 100644 --- a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -27,11 +27,7 @@ public void SerializationContract_IdentifiesCurrentType() Assert.Empty(type.GetCustomAttributes()); var field = Assert.Single(type.GetFields(BindingFlags.Instance | BindingFlags.NonPublic)); Assert.True(field.IsInitOnly); - Assert.True(field.FieldType.IsSealed); - Assert.Equal(typeof(string), field.FieldType.GetProperty("Canonical")!.PropertyType); - Assert.Equal(typeof(int), field.FieldType.GetProperty("Hash")!.PropertyType); - Assert.DoesNotContain(field.FieldType.GetFields(BindingFlags.Instance | BindingFlags.NonPublic), - value => value.FieldType == typeof(HierarchicalKey) || value.FieldType == field.FieldType); + Assert.Equal(typeof(string), field.FieldType); } [Fact] diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs index 6995ba7bea4..ac1c06955a7 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs @@ -205,7 +205,7 @@ public void AddMessageType_ReturnsOriginalCollectionAndRegistersExactKeyedSingle var services = new ServiceCollection(); var result = services.AddMessageType(Subject); Assert.Same(services, result); - Assert.Equal(3, services.Count); + Assert.Equal(2, services.Count); var registration = Assert.Single(services, descriptor => descriptor.IsKeyedService); Assert.True(registration.IsKeyedService); Assert.Equal(Subject, registration.ServiceKey); @@ -498,7 +498,7 @@ public void MessageType_EncodingFailurePreservesOutstandingBytesAndPristineRetry Assert.Equal((byte)0x7e, probe.CommittedFailureByte); var recovered = messageType.Create(MessageKey, Receiver, "recovered 東京 reserve"); - Assert.Equal(2, probe.WriteCount); + Assert.Equal(3, probe.WriteCount); Assert.Equal(pristine.SerializeToArray("recovered 東京 reserve"), recovered.Envelope.Payload.ToArray()); Assert.Equal("recovered 東京 reserve", pristine.Deserialize(recovered.Envelope.Payload)); Assert.Equal(outstandingWire, outstanding.Envelope.Payload.ToArray()); @@ -846,13 +846,10 @@ public async Task InboxDispatcher_HandleAsyncPropagatesDeferredValueTaskCompleti // before assertions, so even assertion failure cannot strand a dispatch. var completedBeforeRelease = dispatch.IsCompleted; var completionsBeforeRelease = context.CompletionCount; - var lateRegistration = Record.Exception(() => dispatcher.Register( - new MessageType("orders.later", serializer), (_, _, _) => ValueTask.CompletedTask)); completion.SetResult(); await dispatch; Assert.False(completedBeforeRelease); - Assert.IsType(lateRegistration); Assert.Equal(0, completionsBeforeRelease); Assert.Equal("deferred €42", capturedBody); Assert.Same(context, capturedContext); @@ -889,13 +886,10 @@ public async Task InboxDispatcher_HandleAsyncPropagatesDeferredValueTaskFault() var dispatch = dispatcher.HandleAsync(context, cancellation.Token); var completedBeforeRelease = dispatch.IsCompleted; var completionsBeforeRelease = context.CompletionCount; - var lateRegistration = Record.Exception(() => dispatcher.Register( - new MessageType("orders.later", serializer), (_, _, _) => ValueTask.CompletedTask)); completion.SetException(sentinel); var exception = await Assert.ThrowsAsync(async () => await dispatch); Assert.False(completedBeforeRelease); - Assert.IsType(lateRegistration); Assert.Equal(0, completionsBeforeRelease); Assert.Same(sentinel, exception); Assert.Equal("deferred fault €42", capturedBody); @@ -1910,8 +1904,8 @@ void Mutate(string body, IInboxHandlerContext caller) var exception = Assert.Throws(() => dispatcher.HandleAsync(context, CancellationToken.None)); - Assert.Contains(absent ? "requires a subject" : "No handler is registered", exception.Message, - StringComparison.Ordinal); + // Handlers match subjects exactly, so an absent subject matches no typed handler. + Assert.Contains("No handler is registered", exception.Message, StringComparison.Ordinal); Assert.Equal(0, probe.ReadCount); Assert.Equal(0, probe.WriteCount); Assert.Equal(0, calls); From 8285327c77e070e295dc141d4a96d905a6859f0c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 14:41:33 -0700 Subject: [PATCH 03/95] feat(messaging): create message types from the serializer service Adds a MessageType(string, Serializer) constructor which resolves the payload serializer from the Orleans serializer service. --- src/Orleans.Messaging/MessageType.cs | 12 +++++++++++ .../Contracts/TypedMessageHelperTests.cs | 20 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Messaging/MessageType.cs b/src/Orleans.Messaging/MessageType.cs index 0a895921ea1..290c71aef36 100644 --- a/src/Orleans.Messaging/MessageType.cs +++ b/src/Orleans.Messaging/MessageType.cs @@ -27,6 +27,18 @@ public MessageType(string subject, Serializer serializer) _serializer = serializer; } + /// Creates a subject and payload-type binding using the payload serializer from . + /// The ordinal protocol subject. + /// The serializer which provides the payload serializer. + /// + /// Use this overload to define a protocol's bindings together in one application class, constructed from the + /// service. + /// + public MessageType(string subject, Serializer serializer) + : this(subject, (serializer ?? throw new ArgumentNullException(nameof(serializer))).GetSerializer()) + { + } + /// Gets the exact protocol subject. public string Subject { get; } diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs index ac1c06955a7..dc58187dba4 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs @@ -28,12 +28,28 @@ public sealed class TypedMessageHelperTests [Fact] public void MessageType_RejectsNullSerializer() { - using var services = CreateServices(); var exception = Assert.Throws( - () => new MessageType(Subject, null!)); + () => new MessageType(Subject, (Serializer)null!)); + Assert.Equal("serializer", exception.ParamName); + + exception = Assert.Throws( + () => new MessageType(Subject, (Serializer)null!)); Assert.Equal("serializer", exception.ParamName); } + [Fact] + public void MessageType_CreatedFromSerializer_RoundTripsBody() + { + using var services = CreateServices(); + var messageType = new MessageType(Subject, services.GetRequiredService()); + + var message = messageType.Create(MessageKey, Receiver, "payload"); + + Assert.Equal(Subject, messageType.Subject); + Assert.Equal(Subject, message.Envelope.Subject); + Assert.Equal("payload", messageType.Decode(message)); + } + [Fact] public void MessageType_RejectsNullSubject() { From cab356f9e810691fc0967388a04fa0e040585074 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 15:36:02 -0700 Subject: [PATCH 04/95] docs(messaging): add Microsoft.Orleans.Messaging to the package inventory Lists the package in the NuGet package reference and records it as awaiting first publication. --- docs/site/src/content/docs/resources/nuget-packages.md | 1 + docs/site/src/data/external-link-allowlist.json | 1 + docs/site/src/data/unpublished-api-packages.json | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/site/src/content/docs/resources/nuget-packages.md b/docs/site/src/content/docs/resources/nuget-packages.md index 6858d4ef0c9..72b0f35264d 100644 --- a/docs/site/src/content/docs/resources/nuget-packages.md +++ b/docs/site/src/content/docs/resources/nuget-packages.md @@ -76,6 +76,7 @@ Memory persistence distributes records across cluster storage grains but isn't d | [Microsoft.Orleans.Reminders.Redis](https://www.nuget.org/packages/Microsoft.Orleans.Reminders.Redis) | Redis reminders. | | [Microsoft.Orleans.DurableJobs](https://www.nuget.org/packages/Microsoft.Orleans.DurableJobs) | Distributed scheduling for durable one-time jobs. | | [Microsoft.Orleans.DurableJobs.AzureStorage](https://www.nuget.org/packages/Microsoft.Orleans.DurableJobs.AzureStorage) | Azure Blob Storage for durable jobs. | +| [Microsoft.Orleans.Messaging](https://www.nuget.org/packages/Microsoft.Orleans.Messaging) | Experimental grain-scoped inbox and outbox messaging. | Use reminders for recurring durable callbacks and durable jobs for scheduled one-time work. Grain timers are activation-scoped and use the core runtime rather than a provider package. diff --git a/docs/site/src/data/external-link-allowlist.json b/docs/site/src/data/external-link-allowlist.json index eee6b628963..f667d1fbc36 100644 --- a/docs/site/src/data/external-link-allowlist.json +++ b/docs/site/src/data/external-link-allowlist.json @@ -11,6 +11,7 @@ "https://docs.aws.amazon.com/sdk-for-net/v4/developer-guide/creds-assign.html": "AWS serves this public SDK credential-resolution page to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", "https://docs.aws.amazon.com/sdkfornet/v4/apidocs/items/Util/TLoggingConfig.html": "AWS serves this public .NET SDK logging configuration reference to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", "https://docs.aws.amazon.com/streams/latest/dev/introduction.html": "AWS serves this public Kinesis overview to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", + "https://www.nuget.org/packages/Microsoft.Orleans.Messaging": "The new Messaging package is documented but not yet published; remove this entry and its unpublished API-package entry after publication.", "https://en.wikipedia.org/wiki/Kalman_filter": "Wikipedia serves this public article to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403.", "https://twitter.com/intent/follow?screen_name=msftorleans": "Twitter serves this public Orleans follow intent to browsers and intermittently returns HTTP 403 to the bounded automated HEAD/GET probe; successful responses also occur.", "https://www.f5.com/company/blog/nginx/nginx-power-of-two-choices-load-balancing-algorithm": "The F5 site serves this canonical NGINX article to browsers but rejects the bounded automated HEAD/GET probe with HTTP 403." diff --git a/docs/site/src/data/unpublished-api-packages.json b/docs/site/src/data/unpublished-api-packages.json index 64d185c77e2..49cf020943e 100644 --- a/docs/site/src/data/unpublished-api-packages.json +++ b/docs/site/src/data/unpublished-api-packages.json @@ -1,4 +1,6 @@ { "description": "Generated API assemblies which are not currently published as standalone NuGet packages.", - "packages": {} + "packages": { + "Microsoft.Orleans.Messaging": "The new package is awaiting its first NuGet publication." + } } From 3e942dcba5678d53c171bd4c1d772de3415ceb4e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 15:43:06 -0700 Subject: [PATCH 05/95] chore(messaging): add generated API surface Generates src/api/Orleans.Messaging and describes the package README as the contract layer. --- src/Orleans.Messaging/README.md | 7 +- .../Orleans.Messaging/Orleans.Messaging.cs | 438 ++++++++++++++++++ 2 files changed, 440 insertions(+), 5 deletions(-) create mode 100644 src/api/Orleans.Messaging/Orleans.Messaging.cs diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 9fd3a2022b7..930eadccaea 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -1,10 +1,7 @@ # Microsoft Orleans Messaging -`Microsoft.Orleans.Messaging` provides grain-scoped durable inboxes and outboxes -for Microsoft Orleans. A grain sends a message by staging it in its outbox; the -receiver's inbox processes it at least once and deduplicates it by message ID. - -This package defines the messaging contracts: envelopes, hierarchical message +`Microsoft.Orleans.Messaging` defines the contracts for grain-scoped durable +inboxes and outboxes in Microsoft Orleans: envelopes, hierarchical message identities, inbox and outbox interfaces, handler registration, typed message bindings, delivery results, and options. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs new file mode 100644 index 00000000000..6de0a1bc996 --- /dev/null +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -0,0 +1,438 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Orleans.Hosting +{ + public static partial class MessageTypeExtensions + { + public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddMessageType(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, string subject) { throw null; } + } +} + +namespace Orleans.Messaging +{ + [GenerateSerializer] + [Alias("Orleans.Messaging.DeliveryResult")] + public readonly partial struct DeliveryResult + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(1)] + public string? Message { get { throw null; } init { } } + + [Id(0)] + public DeliveryStatus Status { get { throw null; } init { } } + + public static DeliveryResult Accepted() { throw null; } + + public static DeliveryResult Backpressured() { throw null; } + + public static DeliveryResult DeadLettered(string reason) { throw null; } + + public static DeliveryResult Duplicate() { throw null; } + + public static DeliveryResult HandlerNotFound() { throw null; } + } + + public enum DeliveryStatus + { + Accepted = 0, + Duplicate = 1, + Backpressured = 2, + HandlerNotFound = 3, + DeadLettered = 4 + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.Envelope")] + public readonly partial struct Envelope + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + [Id(3)] + public System.Collections.Generic.Dictionary? Headers { get { throw null; } init { } } + + [Id(0)] + public required HierarchicalKey MessageId { get { throw null; } init { } } + + [Id(1)] + public required byte[] Payload { get { throw null; } init { } } + + [Id(2)] + public string? Subject { get { throw null; } init { } } + } + + [Immutable] + [Alias("Orleans.Messaging.HierarchicalKey")] + [System.Text.Json.Serialization.JsonConverter(typeof(HierarchicalKeyJsonConverter))] + public readonly partial struct HierarchicalKey : System.ISpanFormattable, System.IFormattable, System.IEquatable, System.IParsable, System.ISpanParsable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public const char EscapeCharacter = '\\'; + public const char SegmentSeparator = '/'; + public bool IsDefault { get { throw null; } } + + public int Length { get { throw null; } } + + public readonly HierarchicalKey Append(HierarchicalKey suffix) { throw null; } + + public static HierarchicalKey Create(scoped params System.ReadOnlySpan values) { throw null; } + + public static HierarchicalKey Create(string value) { throw null; } + + public readonly HierarchicalKey CreateChildKey(string value) { throw null; } + + public readonly bool Equals(HierarchicalKey other) { throw null; } + + public override readonly bool Equals(object? obj) { throw null; } + + public readonly SegmentEnumerator GetEnumerator() { throw null; } + + public override readonly int GetHashCode() { throw null; } + + public readonly HierarchicalKey? GetParent() { throw null; } + + public readonly bool IsAncestorOf(HierarchicalKey other) { throw null; } + + public readonly bool IsChildOf(HierarchicalKey other) { throw null; } + + public readonly bool IsParentOf(HierarchicalKey other) { throw null; } + + public static bool operator ==(HierarchicalKey left, HierarchicalKey right) { throw null; } + + public static bool operator !=(HierarchicalKey left, HierarchicalKey right) { throw null; } + + static HierarchicalKey System.ISpanParsable.Parse(System.ReadOnlySpan s, System.IFormatProvider? provider) { throw null; } + + static HierarchicalKey System.IParsable.Parse(string s, System.IFormatProvider? provider) { throw null; } + + public override readonly string ToString() { throw null; } + + public readonly string ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } + + public readonly bool TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } + + static bool System.ISpanParsable.TryParse(System.ReadOnlySpan s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + static bool System.IParsable.TryParse(string? s, System.IFormatProvider? provider, out HierarchicalKey result) { throw null; } + + public sealed partial class HierarchicalKeyJsonConverter : System.Text.Json.Serialization.JsonConverter + { + public override HierarchicalKey Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override HierarchicalKey ReadAsPropertyName(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override void Write(System.Text.Json.Utf8JsonWriter writer, HierarchicalKey value, System.Text.Json.JsonSerializerOptions options) { } + + public override void WriteAsPropertyName(System.Text.Json.Utf8JsonWriter writer, HierarchicalKey value, System.Text.Json.JsonSerializerOptions options) { } + } + + public ref partial struct SegmentEnumerator + { + private object _dummy; + private int _dummyPrimitive; + public System.ReadOnlySpan Current { get { throw null; } } + + public bool MoveNext() { throw null; } + } + } + + public partial interface IInbox + { + IInbox Register(IInboxHandler handler); + } + + [Alias("IInboxExtension")] + public partial interface IInboxExtension : Runtime.IGrainExtension, Runtime.IAddressable + { + [Alias("DeliverAsync")] + System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); + } + + public partial interface IInboxHandler + { + bool CanHandle(IInboxHandlerContext context); + System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); + } + + public partial interface IInboxHandlerContext + { + InboxMessage Message { get; } + + void Complete(); + void Fail(string reason); + } + + public static partial class InboxExtensions + { + public static IInbox Register(this IInbox inbox, MessageType messageType, System.Action handler) { throw null; } + + public static IInbox Register(this IInbox inbox, MessageType messageType, System.Func handler) { throw null; } + + public static IInbox Register(this IInbox inbox, MessageType messageType, TArg argument, System.Action handler) { throw null; } + + public static IInbox Register(this IInbox inbox, MessageType messageType, TArg argument, System.Func handler) { throw null; } + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.InboxMessage")] + public readonly partial struct InboxMessage + { + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + } + + public partial interface IOutbox + { + void Send(OutboxMessage message); + } + + public sealed partial class MessageType + { + public MessageType(string subject, Serialization.Serializer serializer) { } + + public MessageType(string subject, Serialization.Serializer serializer) { } + + public string Subject { get { throw null; } } + + public OutboxMessage Create(HierarchicalKey messageId, Runtime.GrainId receiverId, T body) { throw null; } + + public T Decode(Envelope envelope) { throw null; } + + public T Decode(InboxMessage message) { throw null; } + + public T Decode(OutboxMessage message) { throw null; } + } + + public static partial class OutboxExtensions + { + public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.GrainId destination, T body) { } + + public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.IAddressable destination, T body) { } + + public static void Send(this IOutbox outbox, MessageType messageType, System.Collections.Generic.IEnumerable<(HierarchicalKey MessageId, Runtime.GrainId Destination, T Body)> messages) { } + + public static void SendReply(this IOutbox outbox, MessageType messageType, IInboxHandlerContext context, Runtime.GrainId destination, T body) { } + + public static void SendReply(this IOutbox outbox, MessageType messageType, IInboxHandlerContext context, Runtime.IAddressable destination, T body) { } + } + + [GenerateSerializer] + [Alias("Orleans.Messaging.OutboxMessage")] + public readonly partial struct OutboxMessage + { + [Id(0)] + public required Envelope Envelope { get { throw null; } init { } } + + [Id(1)] + public required Runtime.GrainId ReceiverId { get { throw null; } init { } } + + public readonly InboxMessage ToInboxMessage() { throw null; } + } +} + +namespace Orleans.Messaging.Configuration +{ + public partial class DurableMessagingOptions + { + public System.TimeSpan BackpressureRetryDelay { get { throw null; } set { } } + + public System.TimeSpan DeadLetterRetentionPeriod { get { throw null; } set { } } + + public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } + + public int InboxBatchSize { get { throw null; } set { } } + + public int MaxCapacity { get { throw null; } set { } } + + public int MaxDeliveryAttempts { get { throw null; } set { } } + + public System.TimeSpan MaxOutboxRetryAge { get { throw null; } set { } } + + public int MaxProcessingAttempts { get { throw null; } set { } } + + public int MaxRetainedDeadLetters { get { throw null; } set { } } + + public int OutboxBatchSize { get { throw null; } set { } } + + public void Validate() { } + } +} + +namespace OrleansCodeGen.Orleans.Messaging +{ + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DeliveryResult : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_DeliveryResult(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.DeliveryResult instance) { } + + public global::Orleans.Messaging.DeliveryResult ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.DeliveryResult instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.DeliveryResult value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Envelope : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_Envelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.Envelope instance) { } + + public global::Orleans.Messaging.Envelope ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.Envelope instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.Envelope value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_InboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.InboxMessage instance) { } + + public global::Orleans.Messaging.InboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.InboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.InboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) { } + + public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_OutboxMessage : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Serializers.IValueSerializer, global::Orleans.Serialization.Serializers.IValueSerializer + { + public Codec_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, scoped ref global::Orleans.Messaging.OutboxMessage instance) { } + + public global::Orleans.Messaging.OutboxMessage ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, scoped ref global::Orleans.Messaging.OutboxMessage instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.OutboxMessage value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Envelope : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Envelope(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.Envelope DeepCopy(global::Orleans.Messaging.Envelope original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_InboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_InboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.InboxMessage DeepCopy(global::Orleans.Messaging.InboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 DeepCopy(Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_OutboxMessage : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_OutboxMessage(global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.OutboxMessage DeepCopy(global::Orleans.Messaging.OutboxMessage original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.Messaging.IInboxExtension), typeof(global::Orleans.Messaging.IInboxExtension), "DeliverAsync" })] + public sealed partial class Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Runtime.Request + { + public global::Orleans.Messaging.InboxMessage arg0; + public System.Threading.CancellationToken arg1; + public override bool IsCancellable { get { throw null; } } + + public override void Dispose() { } + + public override string GetActivityName() { throw null; } + + public override object GetArgument(int index) { throw null; } + + public override int GetArgumentCount() { throw null; } + + public override System.Threading.CancellationToken GetCancellationToken() { throw null; } + + public override string GetInterfaceName() { throw null; } + + public override System.Type GetInterfaceType() { throw null; } + + public override System.Reflection.MethodInfo GetMethod() { throw null; } + + public override string GetMethodName() { throw null; } + + public override object GetTarget() { throw null; } + + protected override System.Threading.Tasks.ValueTask InvokeInner() { throw null; } + + public override void SetArgument(int index, object value) { } + + public override void SetTarget(global::Orleans.Serialization.Invocation.ITargetHolder holder) { } + + public override bool TryCancel() { throw null; } + } +} \ No newline at end of file From 3e34799a89e40727900960a7b603ec64653ec927 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 21:20:22 -0700 Subject: [PATCH 06/95] fix(messaging)!: reserve zero as an invalid delivery status A default DeliveryResult no longer reports Accepted. DeliveryStatus values now start at 1. BREAKING CHANGE: DeliveryStatus numeric values changed (Accepted = 1 through DeadLettered = 5). --- src/Orleans.Messaging/DeliveryStatus.cs | 11 ++++++----- src/api/Orleans.Messaging/Orleans.Messaging.cs | 10 +++++----- .../Contracts/DeliveryAndOptionsContractTests.cs | 3 ++- .../Contracts/EnvelopeContractTests.cs | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Orleans.Messaging/DeliveryStatus.cs b/src/Orleans.Messaging/DeliveryStatus.cs index d4d24333e2d..d12ca7b3a0c 100644 --- a/src/Orleans.Messaging/DeliveryStatus.cs +++ b/src/Orleans.Messaging/DeliveryStatus.cs @@ -3,30 +3,31 @@ namespace Orleans.Messaging; /// /// Status codes for delivery attempts. /// +/// Zero is not a valid status, so a default reports no outcome. public enum DeliveryStatus { /// /// Message was accepted and persisted to inbox. /// - Accepted = 0, + Accepted = 1, /// /// Message was a duplicate (already processed or in inbox). /// - Duplicate = 1, + Duplicate = 2, /// /// Inbox is at capacity; sender should retry later. /// - Backpressured = 2, + Backpressured = 3, /// /// The receiving inbox has no registered handler. /// - HandlerNotFound = 3, + HandlerNotFound = 4, /// /// The message was moved to the receiver's dead-letter store. /// - DeadLettered = 4 + DeadLettered = 5 } diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 6de0a1bc996..31fe9d933c0 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -41,11 +41,11 @@ public readonly partial struct DeliveryResult public enum DeliveryStatus { - Accepted = 0, - Duplicate = 1, - Backpressured = 2, - HandlerNotFound = 3, - DeadLettered = 4 + Accepted = 1, + Duplicate = 2, + Backpressured = 3, + HandlerNotFound = 4, + DeadLettered = 5 } [GenerateSerializer] diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index 0cf2d33aefe..8c3b4a6ad60 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -35,7 +35,8 @@ public void DeliveryStatus_AllValues_HaveContiguousDistinctValues() DeliveryStatus.DeadLettered ], Enum.GetValues()); - Assert.Equal([0, 1, 2, 3, 4], Enum.GetValues().Select(static value => (int)value)); + Assert.Equal([1, 2, 3, 4, 5], Enum.GetValues().Select(static value => (int)value)); + Assert.False(Enum.IsDefined(default(DeliveryResult).Status)); } [Fact] diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 9e06c6f86f9..1e081bc66e1 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -193,7 +193,7 @@ public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.EnvelopeBuilder")); Assert.Equal(typeof(void), typeof(IInboxHandlerContext).GetMethod("Fail")!.ReturnType); Assert.Equal(typeof(string), Assert.Single(typeof(IInboxHandlerContext).GetMethod("Fail")!.GetParameters()).ParameterType); - Assert.Equal(new[] { "Accepted:0", "Backpressured:2", "DeadLettered:4", "Duplicate:1", "HandlerNotFound:3" }, + Assert.Equal(new[] { "Accepted:1", "Backpressured:3", "DeadLettered:5", "Duplicate:2", "HandlerNotFound:4" }, Enum.GetNames().Select(name => $"{name}:{(int)Enum.Parse(name)}").Order(StringComparer.Ordinal)); AssertIds(typeof(DeliveryResult), ("Status", 0u), ("Message", 1u)); using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); From 4635669e33bde3223f69165b5db61cf9228489e7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 21:20:22 -0700 Subject: [PATCH 07/95] fix(messaging): check cancellation before decoding typed inbox bodies Typed handlers receive every decoded body, so the delegate owns resources decoded while cancellation arrives. --- src/Orleans.Messaging/InboxExtensions.cs | 3 +- .../Contracts/TypedMessageHelperTests.cs | 67 ++++++++++++++----- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/Orleans.Messaging/InboxExtensions.cs b/src/Orleans.Messaging/InboxExtensions.cs index 18eb0af1fa6..580fddf18ca 100644 --- a/src/Orleans.Messaging/InboxExtensions.cs +++ b/src/Orleans.Messaging/InboxExtensions.cs @@ -81,8 +81,9 @@ public bool CanHandle(IInboxHandlerContext context) => public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { - var body = messageType.Decode(context.Message); cancellationToken.ThrowIfCancellationRequested(); + // The delegate owns the decoded body, including when cancellation arrives during decoding. + var body = messageType.Decode(context.Message); return Invoke(body, context, cancellationToken); } diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs index dc58187dba4..21d3103f220 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs @@ -1024,6 +1024,44 @@ public async Task InboxDispatcher_HandleAsyncLeavesDecodedResourceApplicationOwn } } + [Fact] + public async Task InboxDispatcher_CancellationDuringDecodeDeliversDecodedResourceToHandler() + { + using var services = CreateServices(); + var codec = new ResourceCodec(); + var serializer = new Serializer(codec, services.GetRequiredService()); + using var original = new OwnedResource("canceled resource €42"); + var envelope = DirectEnvelope(serializer, Subject, original); + var context = new CountingInboxHandlerContext(envelope); + using var cancellation = new CancellationTokenSource(); + codec.OnRead = cancellation.Cancel; + OwnedResource? captured = null; + var observedToken = CancellationToken.None; + var dispatcher = new TestInbox(); + dispatcher.Register(new MessageType(Subject, serializer), (body, _, token) => + { + captured = body; + observedToken = token; + return ValueTask.CompletedTask; + }); + try + { + await dispatcher.HandleAsync(context, cancellation.Token); + + var decoded = Assert.IsType(captured); + Assert.Same(codec.LastDecoded, decoded); + Assert.Equal(1, codec.ReadCount); + Assert.Equal(cancellation.Token, observedToken); + Assert.True(observedToken.IsCancellationRequested); + Assert.Equal(0, decoded.DisposeCount); + Assert.Equal(0, context.CompletionCount); + } + finally + { + codec.LastDecoded?.Dispose(); + } + } + [Fact] public async Task InboxDispatcher_HandleAsyncRejectsSerializedNullBeforeHandler() { @@ -1113,17 +1151,14 @@ public void RegisterAction_PropagatesOriginalDelegateFailure() Assert.Equal(0, context.CompletionCount); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public void RegisterAction_CancellationBeforeEntryLeavesHandlerAndCompletionUntouched(bool cancelDuringDecode) + [Fact] + public void RegisterAction_CancellationBeforeEntryLeavesHandlerAndCompletionUntouched() { using var services = CreateServices(); var direct = services.GetRequiredService>(); var (serializer, probe) = CreateProbedSerializer(services); using var cancellation = new CancellationTokenSource(); - if (cancelDuringDecode) probe.OnRead = cancellation.Cancel; - else cancellation.Cancel(); + cancellation.Cancel(); var envelope = DirectEnvelope(direct, Subject, "cancellation"); var context = new CountingInboxHandlerContext(envelope); var calls = 0; @@ -1134,7 +1169,7 @@ public void RegisterAction_CancellationBeforeEntryLeavesHandlerAndCompletionUnto dispatcher.HandleAsync(context, cancellation.Token)); Assert.Equal(cancellation.Token, exception.CancellationToken); - Assert.Equal(cancelDuringDecode ? 1 : 0, probe.ReadCount); + Assert.Equal(0, probe.ReadCount); Assert.Equal(0, calls); Assert.Equal(0, context.CompletionCount); } @@ -1349,18 +1384,15 @@ public async Task RegisterState_PropagatesOriginalSynchronousFailureWithoutCompl } [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public void RegisterState_CancellationBeforeOrDuringDecodePreventsAllDelegateEffects(bool asynchronous, bool duringDecode) + [InlineData(false)] + [InlineData(true)] + public void RegisterState_CancellationBeforeDecodePreventsAllDelegateEffects(bool asynchronous) { using var services = CreateServices(); var direct = services.GetRequiredService>(); var (serializer, probe) = CreateProbedSerializer(services); using var cancellation = new CancellationTokenSource(); - if (duringDecode) probe.OnRead = cancellation.Cancel; - else cancellation.Cancel(); + cancellation.Cancel(); var envelope = DirectEnvelope(direct, Subject, "canceled state"); var context = new CountingInboxHandlerContext(envelope); var state = new HandlerState { Complete = true }; @@ -1372,7 +1404,7 @@ public void RegisterState_CancellationBeforeOrDuringDecodePreventsAllDelegateEff var exception = Assert.Throws(() => dispatcher.HandleAsync(context, cancellation.Token)); Assert.Equal(cancellation.Token, exception.CancellationToken); - Assert.Equal(duringDecode ? 1 : 0, probe.ReadCount); + Assert.Equal(0, probe.ReadCount); Assert.Equal(0, state.Calls); Assert.Null(state.Body); Assert.Null(state.Context); @@ -1829,6 +1861,7 @@ private sealed class ResourceCodec : IFieldCodec { public OwnedResource? LastDecoded { get; private set; } public int ReadCount { get; private set; } + public Action? OnRead { get; set; } public void WriteField( ref Writer writer, uint fieldIdDelta, [AllowNull] Type expectedType, @@ -1846,7 +1879,9 @@ public OwnedResource ReadValue(ref Reader reader, Field field) ReadCount++; var name = StringCodec.ReadValue(ref reader, field) ?? throw new InvalidDataException("Resource name must not be null."); - return LastDecoded = new OwnedResource(name); + LastDecoded = new OwnedResource(name); + OnRead?.Invoke(); + return LastDecoded; } } From 213c8b43b834dab591d4e92a7d72753ba8477670 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 21:20:23 -0700 Subject: [PATCH 08/95] build(messaging): set the package readme file --- src/Orleans.Messaging/Orleans.Messaging.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj index 69982a6b231..599c7880a8a 100644 --- a/src/Orleans.Messaging/Orleans.Messaging.csproj +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -4,6 +4,7 @@ Microsoft Orleans Messaging Inbox and outbox messaging for Microsoft Orleans. $(PackageTags) Messaging Inbox Outbox + README.md true $(DefaultTargetFrameworks) disable From 174cc069984cc6418bec4ed9d2f50c3dc33a43ca Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 21:28:51 -0700 Subject: [PATCH 09/95] feat(messaging): add Envelope.DeliverAt Adds an optional delivery time to the envelope contract. The receiving inbox processes the message once the Durable Jobs clock reaches DeliverAt. DeliverAt is part of command equivalence, is omitted from JSON when unset, and MessageType.Create and the typed OutboxExtensions.Send overloads accept it as an optional argument. --- src/Orleans.Messaging/Envelope.cs | 21 ++++++++++++++++- src/Orleans.Messaging/EnvelopeEquivalence.cs | 1 + src/Orleans.Messaging/MessageType.cs | 9 ++++++-- src/Orleans.Messaging/OutboxExtensions.cs | 16 +++++++++---- .../Orleans.Messaging/Orleans.Messaging.cs | 10 +++++--- .../Contracts/EnvelopeContractTests.cs | 20 ++++++++++++---- .../Contracts/TypedOutboxExtensionsTests.cs | 23 +++++++++++++++++++ 7 files changed, 86 insertions(+), 14 deletions(-) diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs index 628327040ae..f1b29de853e 100644 --- a/src/Orleans.Messaging/Envelope.cs +++ b/src/Orleans.Messaging/Envelope.cs @@ -1,8 +1,10 @@ +using System; using System.Collections.Generic; +using System.Text.Json.Serialization; namespace Orleans.Messaging; -/// Contains a command identity, payload, subject, and optional application headers. +/// Contains a command identity, payload, subject, optional application headers, and an optional delivery time. /// /// Message identity and payload are required; an empty payload is valid. Raw protocols can omit /// subject and headers. Prepare the payload and headers before publishing a message and keep them @@ -28,4 +30,21 @@ public readonly struct Envelope /// Use an ordinal dictionary for case-sensitive header names. Keys are nonempty and arrays are non-null. [Id(3)] public Dictionary? Headers { get; init; } + + /// Gets the optional earliest time at which the receiving inbox processes the message. + /// + /// + /// The outbox delivers the message immediately. The receiving inbox accepts, deduplicates, and persists it + /// as usual, then holds it until this time is reached on the Durable Jobs clock, and one durable wake-up + /// resumes the grain when the message becomes due. processes the message as soon as + /// its acceptance is durable. + /// + /// + /// The time is part of the command: reusing a pending message ID with a different value is a conflicting command. + /// A message with a value does not count toward the inbox MaxCapacity, including after it becomes due. + /// + /// + [Id(4)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? DeliverAt { get; init; } } diff --git a/src/Orleans.Messaging/EnvelopeEquivalence.cs b/src/Orleans.Messaging/EnvelopeEquivalence.cs index 4710813bacd..bdbf42079f8 100644 --- a/src/Orleans.Messaging/EnvelopeEquivalence.cs +++ b/src/Orleans.Messaging/EnvelopeEquivalence.cs @@ -13,5 +13,6 @@ public static bool AreSameCommand(Envelope left, Envelope right) => left.MessageId == right.MessageId && left.Payload.Length == right.Payload.Length && string.Equals(left.Subject, right.Subject, StringComparison.Ordinal) + && left.DeliverAt == right.DeliverAt && left.Payload.AsSpan().SequenceEqual(right.Payload); } diff --git a/src/Orleans.Messaging/MessageType.cs b/src/Orleans.Messaging/MessageType.cs index 290c71aef36..614d1378aba 100644 --- a/src/Orleans.Messaging/MessageType.cs +++ b/src/Orleans.Messaging/MessageType.cs @@ -78,11 +78,15 @@ public T Decode(Envelope envelope) /// The stable application command identity. /// The destination inbox. /// The nonnull body to serialize. + /// + /// The optional earliest time, on the Durable Jobs clock, at which the receiving inbox processes the message. + /// See . + /// /// An outgoing message with the encoded payload. /// /// Serialization completes before publication. Keep the published payload bytes unchanged. /// - public OutboxMessage Create(HierarchicalKey messageId, GrainId receiverId, T body) + public OutboxMessage Create(HierarchicalKey messageId, GrainId receiverId, T body, DateTimeOffset? deliverAt = null) { if (body is null) throw new ArgumentNullException(nameof(body)); if (receiverId.IsDefault) @@ -97,7 +101,8 @@ public OutboxMessage Create(HierarchicalKey messageId, GrainId receiverId, T bod { MessageId = messageId, Subject = Subject, - Payload = payload + Payload = payload, + DeliverAt = deliverAt }, ReceiverId = receiverId }; diff --git a/src/Orleans.Messaging/OutboxExtensions.cs b/src/Orleans.Messaging/OutboxExtensions.cs index da7a623d6e8..16b4a39b786 100644 --- a/src/Orleans.Messaging/OutboxExtensions.cs +++ b/src/Orleans.Messaging/OutboxExtensions.cs @@ -14,6 +14,10 @@ public static class OutboxExtensions /// The stable application command identity. /// The destination inbox. /// The body to encode. + /// + /// The optional earliest time, on the Durable Jobs clock, at which the receiving inbox processes the message. + /// The outbox still delivers the message immediately. See . + /// /// /// Serialization finishes before staging begins. Compute fallible business results first, /// then send, apply the business state, and complete in one synchronous final block. @@ -21,11 +25,11 @@ public static class OutboxExtensions /// public static void Send( this IOutbox outbox, MessageType messageType, - HierarchicalKey messageId, GrainId destination, T body) + HierarchicalKey messageId, GrainId destination, T body, DateTimeOffset? deliverAt = null) { ArgumentNullException.ThrowIfNull(outbox); ArgumentNullException.ThrowIfNull(messageType); - var message = messageType.Create(messageId, destination, body); + var message = messageType.Create(messageId, destination, body, deliverAt); outbox.Send(message); } @@ -36,12 +40,16 @@ public static void Send( /// The stable application command identity. /// The destination grain. /// The body to encode. + /// + /// The optional earliest time, on the Durable Jobs clock, at which the receiving inbox processes the message. + /// The outbox still delivers the message immediately. See . + /// public static void Send( this IOutbox outbox, MessageType messageType, - HierarchicalKey messageId, IAddressable destination, T body) + HierarchicalKey messageId, IAddressable destination, T body, DateTimeOffset? deliverAt = null) { ArgumentNullException.ThrowIfNull(destination); - outbox.Send(messageType, messageId, destination.GetGrainId(), body); + outbox.Send(messageType, messageId, destination.GetGrainId(), body, deliverAt); } /// Encodes and synchronously stages typed commands in enumeration order. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 31fe9d933c0..413987be4bb 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -54,6 +54,10 @@ public readonly partial struct Envelope { private readonly object _dummy; private readonly int _dummyPrimitive; + [Id(4)] + [System.Text.Json.Serialization.JsonIgnore(Condition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)] + public System.DateTimeOffset? DeliverAt { get { throw null; } init { } } + [Id(3)] public System.Collections.Generic.Dictionary? Headers { get { throw null; } init { } } @@ -201,7 +205,7 @@ public MessageType(string subject, Serialization.Serializer serializer) { } public string Subject { get { throw null; } } - public OutboxMessage Create(HierarchicalKey messageId, Runtime.GrainId receiverId, T body) { throw null; } + public OutboxMessage Create(HierarchicalKey messageId, Runtime.GrainId receiverId, T body, System.DateTimeOffset? deliverAt = null) { throw null; } public T Decode(Envelope envelope) { throw null; } @@ -212,9 +216,9 @@ public MessageType(string subject, Serialization.Serializer serializer) { } public static partial class OutboxExtensions { - public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.GrainId destination, T body) { } + public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.GrainId destination, T body, System.DateTimeOffset? deliverAt = null) { } - public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.IAddressable destination, T body) { } + public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.IAddressable destination, T body, System.DateTimeOffset? deliverAt = null) { } public static void Send(this IOutbox outbox, MessageType messageType, System.Collections.Generic.IEnumerable<(HierarchicalKey MessageId, Runtime.GrainId Destination, T Body)> messages) { } diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 1e081bc66e1..ad77040d80b 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -180,7 +180,7 @@ public void MessageAndDeadLetterCopyPreservesMetadataAndIsolation(string kind) [Fact] public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() { - AssertIds(typeof(Envelope), ("MessageId", 0u), ("Payload", 1u), ("Subject", 2u), ("Headers", 3u)); + AssertIds(typeof(Envelope), ("MessageId", 0u), ("Payload", 1u), ("Subject", 2u), ("Headers", 3u), ("DeliverAt", 4u)); AssertIds(typeof(InboxMessage), ("Envelope", 0u)); AssertIds(typeof(OutboxMessage), ("Envelope", 0u), ("ReceiverId", 1u)); Assert.Equal(typeof(Dictionary), typeof(Envelope).GetProperty("Headers")!.PropertyType); @@ -188,7 +188,8 @@ public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() Assert.Null(typeof(Envelope).GetCustomAttribute()); Assert.Single(typeof(Envelope).GetCustomAttributes()); Assert.Equal("Orleans.Messaging.Envelope", typeof(Envelope).GetCustomAttribute()!.Alias); - Assert.Equal(new[] { "Headers", "MessageId", "Payload", "Subject" }, typeof(Envelope).GetProperties().Select(p => p.Name).Order(StringComparer.Ordinal)); + Assert.Equal(typeof(DateTimeOffset?), typeof(Envelope).GetProperty("DeliverAt")!.PropertyType); + Assert.Equal(new[] { "DeliverAt", "Headers", "MessageId", "Payload", "Subject" }, typeof(Envelope).GetProperties().Select(p => p.Name).Order(StringComparer.Ordinal)); Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.PackedEnvelopeHeaders")); Assert.Null(typeof(Envelope).Assembly.GetType("Orleans.Messaging.EnvelopeBuilder")); Assert.Equal(typeof(void), typeof(IInboxHandlerContext).GetMethod("Fail")!.ReturnType); @@ -215,9 +216,13 @@ public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() [InlineData("no-subject", false, false)] [InlineData("payload", false, false)] [InlineData("payload-length", false, false)] + [InlineData("deliver-at-offset", true, true)] + [InlineData("deliver-at", false, false)] + [InlineData("no-deliver-at", false, false)] public void PendingCommandEqualityIgnoresExtraHeadersAndSender(string change, bool outbox, bool inbox) { - var envelope = Create([0, 255, 128]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "first"u8.ToArray() } }; + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.Zero); + var envelope = Create([0, 255, 128]) with { Subject = "subject.v1", Headers = new() { ["custom"] = "first"u8.ToArray() }, DeliverAt = deliverAt }; var original = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; var repeated = new OutboxMessage { @@ -227,7 +232,14 @@ public void PendingCommandEqualityIgnoresExtraHeadersAndSender(string change, bo MessageId = change == "id" ? HierarchicalKey.Create("other") : change == "child" ? Id.CreateChildKey("child") : Id, Payload = change == "payload" ? [0, 255, 129] : change == "payload-length" ? [0, 255] : [0, 255, 128], Subject = change == "subject" ? "other" : change == "subject-case" ? "Subject.v1" : change == "no-subject" ? null : "subject.v1", - Headers = new() { ["custom"] = Encoding.UTF8.GetBytes(change == "headers" ? "second" : "first"), ["sender"] = change == "sender" ? [2] : [1] } + Headers = new() { ["custom"] = Encoding.UTF8.GetBytes(change == "headers" ? "second" : "first"), ["sender"] = change == "sender" ? [2] : [1] }, + DeliverAt = change switch + { + "deliver-at-offset" => deliverAt.ToOffset(TimeSpan.FromHours(2)), + "deliver-at" => deliverAt.AddTicks(1), + "no-deliver-at" => null, + _ => deliverAt + } } }; Assert.Equal(outbox, Invoke("EnvelopeEquivalence", "AreEquivalent", original, repeated)); diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs index 95adc5fd79d..37b7c565886 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs @@ -183,6 +183,29 @@ public void Send_SerializesExactEnvelopeWithExplicitDestination(bool addressable AssertEnvelopeFields(staged.Envelope); Assert.True(outbox.TryGetMessage(CommandId, out var found)); Assert.Same(staged.Envelope.Payload, found.Envelope.Payload); + Assert.Null(staged.Envelope.DeliverAt); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Send_CarriesOptionalDeliverAt(bool addressable) + { + using var services = CreateServices(); + var type = new MessageType(Subject, services.GetRequiredService>()); + var outbox = new StoringOutbox(); + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.FromHours(1)); + + if (addressable) outbox.Send(type, CommandId, new AddressableDestination(Destination), "scheduled", deliverAt); + else outbox.Send(type, CommandId, Destination, "scheduled", deliverAt); + + var staged = Assert.Single(outbox.Messages); + Assert.Equal(deliverAt, staged.Envelope.DeliverAt); + Assert.Equal(deliverAt.Offset, staged.Envelope.DeliverAt!.Value.Offset); + Assert.Equal(Destination, staged.ReceiverId); + Assert.Equal("scheduled", type.Decode(staged)); + Assert.Equal(deliverAt, type.Create(CommandId, Destination, "scheduled", deliverAt).Envelope.DeliverAt); + Assert.Null(type.Create(CommandId, Destination, "scheduled").Envelope.DeliverAt); } [Theory] From a8a766b91be7d26264935caca12b800053c33104 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 23:13:22 -0700 Subject: [PATCH 10/95] docs(messaging): align capacity and pending-intent contracts with DeliverAt MaxCapacity bounds pending inbox messages without DeliverAt, and the inbox admits scheduled messages regardless of it. Pending outbox intent equality includes DeliverAt. --- .../Configuration/DurableMessagingOptions.cs | 5 +++-- src/Orleans.Messaging/IOutbox.cs | 2 +- src/Orleans.Messaging/README.md | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index 82c7d025f72..f2d5b6b43ba 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -29,8 +29,9 @@ public class DurableMessagingOptions / MaximumBackoffMultiplier); /// - /// Gets or sets the maximum number of pending messages in the inbox. - /// When this limit is reached, new message deliveries will return DeliveryResult.Backpressured(). + /// Gets or sets the maximum number of pending inbox messages without . + /// When this limit is reached, new deliveries without return DeliveryResult.Backpressured(). + /// The inbox admits scheduled messages regardless of this limit. /// /// /// diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index 2103b548fff..a33e820d4f1 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -21,7 +21,7 @@ public interface IOutbox /// /// Direct sends share the payload array, header dictionary, and header value arrays. Keep these objects unchanged after publication. /// Equivalent repeated identities retain the original message and headers; payload, optional subject, - /// and destination must match. Custom headers do not change pending intent equality. Conflicts fail explicitly. + /// optional , and destination must match. Custom headers do not change pending intent equality. Conflicts fail explicitly. /// Inbox handlers stage outgoing /// messages in their synchronous final block before calling /// or . diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 930eadccaea..d0327e31391 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -66,7 +66,8 @@ Each receiver returns a `DeliveryResult`: - `Accepted`: the message is durable in the inbox. - `Duplicate`: the message ID is pending or completed within `DeduplicationWindow`. -- `Backpressured`: the inbox holds `MaxCapacity` pending messages. +- `Backpressured`: the inbox holds `MaxCapacity` pending messages without `DeliverAt`. + The inbox admits scheduled messages regardless of `MaxCapacity`. - `HandlerNotFound`: the receiver has no registered handlers. - `DeadLettered`: the message moved to the receiver's dead letters. From 1db3d1c0a108aafdf44d8404cccbf08694c3258c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 23:13:23 -0700 Subject: [PATCH 11/95] test(messaging): cover DeliverAt wire round trips and HierarchicalKey JSON --- .../Contracts/EnvelopeContractTests.cs | 24 +++++++ .../Contracts/HierarchicalKeyTests.cs | 67 +++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index ad77040d80b..18338088159 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -129,6 +129,30 @@ public void OrdinaryCopyAndIndependentWireRoundTrip(int length) Assert.Equal(wire, serializer.SerializeToArray(envelope)); } + [Fact] + public void DeliverAtRoundTripsThroughCreateAndIndependentWire() + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.FromHours(2)); + var type = new MessageType("orders.v1", sending.GetRequiredService>()); + var outgoing = type.Create(Id, Receiver, "scheduled", deliverAt); + Assert.Equal(deliverAt, outgoing.Envelope.DeliverAt); + Validate(outgoing); + + var wire = sending.GetRequiredService>().SerializeToArray(outgoing); + var decoded = receiving.GetRequiredService>().Deserialize(wire); + var copied = sending.GetRequiredService().Copy(outgoing); + foreach (var actual in new[] { decoded, copied }) + { + Assert.Equal(deliverAt, actual.Envelope.DeliverAt); + Assert.Equal(deliverAt.Offset, actual.Envelope.DeliverAt!.Value.Offset); + Assert.Equal(Receiver, actual.ReceiverId); + Assert.Equal("scheduled", type.Decode(actual)); + Assert.True(Invoke("EnvelopeEquivalence", "AreEquivalent", outgoing, actual)); + } + } + [Fact] public void PayloadUsesOrdinaryExternallyBoundOrleansSerializer() { diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs index 298dd156c77..cb82497d4f4 100644 --- a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Reflection; using System.Runtime.CompilerServices; +using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Orleans.Messaging.Tests.Support; using Orleans.Serialization; @@ -30,6 +31,72 @@ public void SerializationContract_IdentifiesCurrentType() Assert.Equal(typeof(string), field.FieldType); } + [Fact] + public void Json_ValueRoundTripsCanonicalEscapedPath() + { + var key = HierarchicalKey.Create("orders", "a/b", @"c\d"); + + var json = JsonSerializer.Serialize(key); + + Assert.Equal(JsonSerializer.Serialize(@"orders/a\/b/c\\d"), json); + var decoded = JsonSerializer.Deserialize(json); + Assert.Equal(key, decoded); + Assert.Equal(3, decoded.CountSegments()); + } + + [Fact] + public void Json_DefaultValueRoundTripsAsNull() + { + Assert.Equal("null", JsonSerializer.Serialize(default(HierarchicalKey))); + Assert.True(JsonSerializer.Deserialize("null").IsDefault); + } + + [Fact] + public void Json_DictionaryKeyRoundTripsCanonicalEscapedPath() + { + var key = HierarchicalKey.Create("orders", "a/b", @"c\d"); + var values = new Dictionary { [key] = 42, [HierarchicalKey.Create("other")] = 7 }; + + var json = JsonSerializer.Serialize(values); + var decoded = JsonSerializer.Deserialize>(json)!; + + Assert.Equal(JsonSerializer.Serialize(new Dictionary { [key.ToString()] = 42, ["other"] = 7 }), json); + Assert.Equal(2, decoded.Count); + Assert.Equal(42, decoded[key]); + Assert.Equal(7, decoded[HierarchicalKey.Create("other")]); + } + + [Fact] + public void Json_DefaultDictionaryKeyIsRejected() + { + var values = new Dictionary { [default] = 1 }; + + Assert.Throws(() => JsonSerializer.Serialize(values)); + } + + [Theory] + [InlineData("\"\"")] + [InlineData("\"a//b\"")] + [InlineData("\"/a\"")] + [InlineData("\"a/\"")] + [InlineData(@"""a\\x""")] + [InlineData(@"""a\\""")] + [InlineData("42")] + [InlineData("{}")] + public void Json_MalformedValueIsRejected(string json) + { + Assert.Throws(() => JsonSerializer.Deserialize(json)); + } + + [Theory] + [InlineData("{\"\":1}")] + [InlineData("{\"a//b\":1}")] + [InlineData(@"{""a\\x"":1}")] + public void Json_MalformedDictionaryKeyIsRejected(string json) + { + Assert.Throws(() => JsonSerializer.Deserialize>(json)); + } + [Fact] public void Default_IsDistinguishedUnsetValue() { From 16483b63fad27bcabbee1b85bfa07af9494fcca9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 10:05:29 -0700 Subject: [PATCH 12/95] feat(messaging): add outbox message source and inbox message sink contracts IOutboxMessageSource provides the envelopes of messages staged with IOutbox.SendFromSource, so the payload lives only in application-owned journaled state. IInboxMessageSink accepts delivered messages directly into application-owned journaled state. IOutbox.Register and IInbox.Register add sources and sinks. --- src/Orleans.Messaging/IInbox.cs | 9 +++ src/Orleans.Messaging/IInboxMessageSink.cs | 40 ++++++++++++ src/Orleans.Messaging/IOutbox.cs | 29 +++++++++ src/Orleans.Messaging/IOutboxMessageSource.cs | 64 +++++++++++++++++++ src/Orleans.Messaging/README.md | 21 +++++- .../Orleans.Messaging/Orleans.Messaging.cs | 21 ++++++ .../Contracts/TypedOutboxExtensionsTests.cs | 4 ++ .../Support/TestInbox.cs | 2 + 8 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 src/Orleans.Messaging/IInboxMessageSink.cs create mode 100644 src/Orleans.Messaging/IOutboxMessageSource.cs diff --git a/src/Orleans.Messaging/IInbox.cs b/src/Orleans.Messaging/IInbox.cs index 840433ba9b2..b00d03607cb 100644 --- a/src/Orleans.Messaging/IInbox.cs +++ b/src/Orleans.Messaging/IInbox.cs @@ -20,4 +20,13 @@ public interface IInbox /// The handler. /// This inbox. IInbox Register(IInboxHandler handler); + + /// Registers a sink which accepts messages directly into application-owned journaled state. + /// The sink. + /// This inbox. + /// + /// Messages are offered to sinks before inbox acceptance and handler dispatch; see . + /// Register sinks during construction or activation setup, before the grain receives messages. + /// + IInbox Register(IInboxMessageSink sink); } diff --git a/src/Orleans.Messaging/IInboxMessageSink.cs b/src/Orleans.Messaging/IInboxMessageSink.cs new file mode 100644 index 00000000000..cb8f9667dbe --- /dev/null +++ b/src/Orleans.Messaging/IInboxMessageSink.cs @@ -0,0 +1,40 @@ +namespace Orleans.Messaging; + +/// +/// Accepts delivered messages directly into application-owned journaled state, so their payload is journaled once. +/// +/// +/// +/// After validation and deduplication, the inbox offers each message without to the +/// registered sinks in registration order, including messages the grain sends to itself. A sink which returns +/// has synchronously staged the message's effects in its own journaled state. The inbox then +/// journals only the message's processed record, without its payload, in the same journal write, and reports +/// once that write is durable. Run effects which depend on the write, such as +/// starting work, from Orleans.Journaling.IStateMachine.OnWriteCompleted. +/// +/// +/// A sink-accepted message never occupies inbox capacity and has no inbox retry or dead-letter lifecycle: the sink +/// owns its processing. Messages which every sink declines, and messages with , are +/// accepted by the inbox and dispatched to instances as usual. +/// +/// +/// Sinks must be idempotent. The inbox deduplicates the message ID only while its processed record is retained for +/// DeduplicationWindow; a later redelivery is offered again. A message whose journal write fails is +/// redelivered to the next activation, which recovers the sink's state from the journal. +/// +/// +public interface IInboxMessageSink +{ + /// Accepts a message into the sink's journaled state, or declines it. + /// The validated, deduplicated message. + /// + /// when the sink staged the message's effects; when it declines the + /// message without staging anything. + /// + /// + /// Runs synchronously on the activation scheduler. Validate the message before staging any change. A sink which + /// throws must leave no staged change: the inbox dead-letters the message with its payload, records its ID as + /// processed, and reports . + /// + bool TryAccept(InboxMessage message); +} diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index a33e820d4f1..7ad1aa10b8a 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -1,4 +1,5 @@ using System; +using Orleans.Runtime; namespace Orleans.Messaging; @@ -33,4 +34,32 @@ public interface IOutbox /// The identity conflicts, or the outbox is unavailable. /// void Send(OutboxMessage message); + + /// + /// Synchronously stages, for the grain's next journal write, a message whose envelope a registered + /// provides. + /// + /// The message ID, which a registered source provides. + /// The receiving grain, which must not be the sending grain. + /// + /// The outbox journals the message ID, receiver, and delivery state, but not the payload, and delivers the message + /// like one staged through . Stage the payload in the source's state before this + /// call: the outbox resolves the source now and fails when none provides the message. Sending a pending ID again + /// with the same receiver coalesces with the pending message. A message addressed to the sending grain is rejected; + /// send it through , which stages it in the grain's own inbox. + /// + /// The ID is unset or too long, or the receiver is default or the sending grain. + /// + /// The ID is pending as a different message, no registered source provides it, or the outbox is unavailable. + /// + void SendFromSource(HierarchicalKey messageId, GrainId receiverId); + + /// Registers a source which provides the envelopes of messages sent through . + /// The source. + /// This outbox. + /// + /// Register sources during construction or activation setup, before journal recovery; see + /// . + /// + IOutbox Register(IOutboxMessageSource source); } diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs new file mode 100644 index 00000000000..913c750e92b --- /dev/null +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -0,0 +1,64 @@ +namespace Orleans.Messaging; + +/// +/// Provides the envelopes of outgoing messages whose payload the application stores in its own journaled state. +/// +/// +/// +/// stages a reference entry which carries the message ID, receiver, and delivery +/// state, but no payload, so the payload is journaled once, in the application's state. The outbox keeps every +/// delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, +/// dead-lettering after MaxDeliveryAttempts or MaxOutboxRetryAge, wake-ups, and metrics. +/// +/// +/// The outbox resolves each reference entry to the first registered source whose returns +/// , when the message is sent and when the outbox recovers it, and calls +/// on every delivery attempt. When the entry retires, the outbox calls +/// exactly once, synchronously, while it stages the entry's removal, so changes the source +/// stages in its journaled state commit in the same journal write as that removal. Register sources during activation +/// setup, before journal recovery: a recovered reference entry which no source can provide fails the activation. +/// +/// +/// All calls run on the activation scheduler. The source keeps the message available from the send until +/// , and returns the same command for every call. +/// +/// +public interface IOutboxMessageSource +{ + /// Determines whether this source provides the message. + /// The ID of a reference entry. + /// when this source provides the message. + bool CanProvide(HierarchicalKey messageId); + + /// Materializes the envelope of a message for a delivery attempt or a dead letter. + /// The ID of a reference entry which this source provides. + /// A valid envelope whose is . + /// + /// The outbox validates the envelope. An exception or an invalid envelope breaks the source's contract and fails the + /// activation's messaging, which deactivates the grain. + /// + Envelope GetEnvelope(HierarchicalKey messageId); + + /// Notifies the source that the outbox retired a message, so the source can release its payload. + /// The ID of the retired reference entry. + /// Whether the receiver retired the message or the message was dead-lettered. + /// + /// Stage the release synchronously in journaled state; it commits in the journal write which removes the entry. + /// A dead-lettered message's envelope is materialized into the outbox dead letter before this call. + /// An exception fails the activation's messaging, which deactivates the grain. + /// + void OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement); +} + +/// Describes how an outbox message retired. +public enum OutboxMessageRetirement +{ + /// The receiver durably accepted the message, or reported it as a duplicate. + Delivered = 1, + + /// + /// The receiver dead-lettered the message, or the outbox dead-lettered it after MaxDeliveryAttempts or once + /// it was older than MaxOutboxRetryAge. + /// + DeadLettered = 2 +} diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index d0327e31391..3ac4de6cccb 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -2,8 +2,8 @@ `Microsoft.Orleans.Messaging` defines the contracts for grain-scoped durable inboxes and outboxes in Microsoft Orleans: envelopes, hierarchical message -identities, inbox and outbox interfaces, handler registration, typed message -bindings, delivery results, and options. +identities, inbox and outbox interfaces, handler, sink, and source registration, +typed message bindings, delivery results, and options. ## Messages @@ -60,11 +60,26 @@ A handler finishes each message with `IInboxHandlerContext.Complete()` or `MessageType.Create` and `OutboxMessage.ToInboxMessage()` build messages for direct inbox delivery. +## Journal payloads once + +When a payload already lives in the grain's own journaled state, an outbox source or +an inbox sink keeps that state as its only journaled copy. Register both during +construction or activation setup, before journal recovery. + +- `IOutbox.Register(IOutboxMessageSource)` adds a source, and + `IOutbox.SendFromSource(messageId, receiverId)` stages a reference entry with the + message ID, receiver, and delivery state, but no payload. The source provides each + envelope through `GetEnvelope`, and `OnRetired` reports + `OutboxMessageRetirement.Delivered` or `DeadLettered`. +- `IInbox.Register(IInboxMessageSink)` adds a sink. A sink whose `TryAccept` returns + `true` has staged the message in its own journaled state. Sinks are idempotent. + ## Delivery results Each receiver returns a `DeliveryResult`: -- `Accepted`: the message is durable in the inbox. +- `Accepted`: the message is durable in the inbox, or, for a message an inbox sink + accepted, in the sink's state together with its processed record. - `Duplicate`: the message ID is pending or completed within `DeduplicationWindow`. - `Backpressured`: the inbox holds `MaxCapacity` pending messages without `DeliverAt`. The inbox admits scheduled messages regardless of `MaxCapacity`. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 413987be4bb..ed717560510 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -150,6 +150,7 @@ public ref partial struct SegmentEnumerator public partial interface IInbox { IInbox Register(IInboxHandler handler); + IInbox Register(IInboxMessageSink sink); } [Alias("IInboxExtension")] @@ -173,6 +174,11 @@ public partial interface IInboxHandlerContext void Fail(string reason); } + public partial interface IInboxMessageSink + { + bool TryAccept(InboxMessage message); + } + public static partial class InboxExtensions { public static IInbox Register(this IInbox inbox, MessageType messageType, System.Action handler) { throw null; } @@ -194,7 +200,16 @@ public readonly partial struct InboxMessage public partial interface IOutbox { + IOutbox Register(IOutboxMessageSource source); void Send(OutboxMessage message); + void SendFromSource(HierarchicalKey messageId, Runtime.GrainId receiverId); + } + + public partial interface IOutboxMessageSource + { + bool CanProvide(HierarchicalKey messageId); + Envelope GetEnvelope(HierarchicalKey messageId); + void OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement); } public sealed partial class MessageType @@ -239,6 +254,12 @@ public readonly partial struct OutboxMessage public readonly InboxMessage ToInboxMessage() { throw null; } } + + public enum OutboxMessageRetirement + { + Delivered = 1, + DeadLettered = 2 + } } namespace Orleans.Messaging.Configuration diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs index 37b7c565886..05a7e3d53cb 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs @@ -569,6 +569,10 @@ public void Send(OutboxMessage message) _messages.Add(message); } + public void SendFromSource(HierarchicalKey messageId, GrainId receiverId) => throw new NotSupportedException(); + + public IOutbox Register(IOutboxMessageSource source) => throw new NotSupportedException(); + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) { foreach (var message in _messages) diff --git a/test/Orleans.Messaging.Tests/Support/TestInbox.cs b/test/Orleans.Messaging.Tests/Support/TestInbox.cs index d5808df5f55..2429da08869 100644 --- a/test/Orleans.Messaging.Tests/Support/TestInbox.cs +++ b/test/Orleans.Messaging.Tests/Support/TestInbox.cs @@ -13,6 +13,8 @@ public IInbox Register(IInboxHandler handler) return this; } + public IInbox Register(IInboxMessageSink sink) => throw new NotSupportedException("The test inbox dispatches to handlers only."); + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => _dispatcher.HandleAsync(context, cancellationToken); } From 3c6ac54cd5ca4af4dd7afc19aab0ff929ecc1db8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 11:04:31 -0700 Subject: [PATCH 13/95] feat(messaging): send source-provided messages by ID and let sources withdraw them IOutbox.Send(HierarchicalKey) replaces SendFromSource, and IOutboxMessageSource.TryGetMessage replaces CanProvide and GetEnvelope. The source supplies the whole OutboxMessage; returning false withdraws a pending message. --- src/Orleans.Messaging/IInbox.cs | 2 +- src/Orleans.Messaging/IOutbox.cs | 26 ++++----- src/Orleans.Messaging/IOutboxMessageSource.cs | 54 +++++++++++-------- src/Orleans.Messaging/README.md | 9 ++-- .../Orleans.Messaging/Orleans.Messaging.cs | 5 +- .../Contracts/TypedOutboxExtensionsTests.cs | 2 +- 6 files changed, 54 insertions(+), 44 deletions(-) diff --git a/src/Orleans.Messaging/IInbox.cs b/src/Orleans.Messaging/IInbox.cs index b00d03607cb..cc5142e16d4 100644 --- a/src/Orleans.Messaging/IInbox.cs +++ b/src/Orleans.Messaging/IInbox.cs @@ -9,7 +9,7 @@ namespace Orleans.Messaging; /// /// A handler decodes the body, performs asynchronous I/O, validates local results, and checks cancellation before /// the first shared business or journaled mutation. From that mutation until it returns, it executes synchronously: -/// it applies safe-to-commit business changes, stages outgoing messages through , and calls +/// it applies safe-to-commit business changes, stages outgoing messages through , and calls /// or . The runtime persists the /// outcome with those changes. Failures before a terminal operation follow the inbox retry and dead-letter policy. /// diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index 7ad1aa10b8a..f25ac27864e 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -1,5 +1,4 @@ using System; -using Orleans.Runtime; namespace Orleans.Messaging; @@ -36,25 +35,28 @@ public interface IOutbox void Send(OutboxMessage message); /// - /// Synchronously stages, for the grain's next journal write, a message whose envelope a registered + /// Synchronously stages, for the grain's next journal write, a message which a registered /// provides. /// /// The message ID, which a registered source provides. - /// The receiving grain, which must not be the sending grain. /// - /// The outbox journals the message ID, receiver, and delivery state, but not the payload, and delivers the message - /// like one staged through . Stage the payload in the source's state before this - /// call: the outbox resolves the source now and fails when none provides the message. Sending a pending ID again - /// with the same receiver coalesces with the pending message. A message addressed to the sending grain is rejected; - /// send it through , which stages it in the grain's own inbox. + /// The outbox journals the message ID and delivery state, while the source holds the payload and receiver, and + /// delivers the message like one staged through . Stage the message in the source's + /// state before this call: the outbox resolves the source now, reads and validates its message, and fails when none + /// provides it. Sending a pending ID again coalesces with the pending message. A message which a source addresses to + /// the sending grain is rejected; send it through , which stages it in the grain's + /// own inbox. /// - /// The ID is unset or too long, or the receiver is default or the sending grain. + /// + /// The ID is unset or too long, or the source's message is invalid or addressed to the sending grain. + /// /// - /// The ID is pending as a different message, no registered source provides it, or the outbox is unavailable. + /// The ID is pending as an ordinary message, no registered source provides it, the source returned a different ID, + /// or the outbox is unavailable. /// - void SendFromSource(HierarchicalKey messageId, GrainId receiverId); + void Send(HierarchicalKey messageId); - /// Registers a source which provides the envelopes of messages sent through . + /// Registers a source which provides messages sent through . /// The source. /// This outbox. /// diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index 913c750e92b..53abb656441 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -1,50 +1,58 @@ namespace Orleans.Messaging; /// -/// Provides the envelopes of outgoing messages whose payload the application stores in its own journaled state. +/// Provides outgoing messages whose payload and receiver the application stores in its own journaled state. /// /// /// -/// stages a reference entry which carries the message ID, receiver, and delivery -/// state, but no payload, so the payload is journaled once, in the application's state. The outbox keeps every -/// delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, +/// stages a reference entry which carries the message ID and delivery +/// state, but no payload or receiver, so the payload is journaled once, in the application's state. The outbox keeps +/// every delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, /// dead-lettering after MaxDeliveryAttempts or MaxOutboxRetryAge, wake-ups, and metrics. /// /// -/// The outbox resolves each reference entry to the first registered source whose returns -/// , when the message is sent and when the outbox recovers it, and calls -/// on every delivery attempt. When the entry retires, the outbox calls -/// exactly once, synchronously, while it stages the entry's removal, so changes the source -/// stages in its journaled state commit in the same journal write as that removal. Register sources during activation -/// setup, before journal recovery: a recovered reference entry which no source can provide fails the activation. +/// The outbox calls when the message is sent and before every delivery attempt. The first +/// registered source which returns the message owns it for the activation's lifetime. When the entry retires, the +/// outbox calls the owner's exactly once, synchronously, while it stages the entry's removal, so +/// changes the source stages in its journaled state commit in the same journal write as that removal. /// /// -/// All calls run on the activation scheduler. The source keeps the message available from the send until -/// , and returns the same command for every call. +/// A source withdraws a pending message by returning from , for +/// example after the application times it out or an administrator removes it. Before its next delivery attempt, the +/// outbox removes the withdrawn entry without delivering it or calling . An attempt already in +/// flight still completes, so a source can receive for a message it withdrew. +/// +/// +/// All calls run on the activation scheduler. Register sources during activation setup, before journal recovery: a +/// recovered reference entry fails the activation when no source is registered. While a source provides a message, it +/// returns the same message, envelope and receiver alike, for every call. /// /// public interface IOutboxMessageSource { - /// Determines whether this source provides the message. + /// Materializes a message for its send or a delivery attempt. /// The ID of a reference entry. - /// when this source provides the message. - bool CanProvide(HierarchicalKey messageId); - - /// Materializes the envelope of a message for a delivery attempt or a dead letter. - /// The ID of a reference entry which this source provides. - /// A valid envelope whose is . + /// + /// When this method returns , a valid message whose envelope's + /// is and whose receiver is another grain. + /// + /// + /// when this source provides the message; when it doesn't provide the + /// message or has withdrawn it. + /// /// - /// The outbox validates the envelope. An exception or an invalid envelope breaks the source's contract and fails the - /// activation's messaging, which deactivates the grain. + /// The outbox validates the message. During , an invalid message fails the + /// send. Afterwards, an exception or an invalid message breaks the source's contract and fails the activation's + /// messaging, which deactivates the grain. /// - Envelope GetEnvelope(HierarchicalKey messageId); + bool TryGetMessage(HierarchicalKey messageId, out OutboxMessage message); /// Notifies the source that the outbox retired a message, so the source can release its payload. /// The ID of the retired reference entry. /// Whether the receiver retired the message or the message was dead-lettered. /// /// Stage the release synchronously in journaled state; it commits in the journal write which removes the entry. - /// A dead-lettered message's envelope is materialized into the outbox dead letter before this call. + /// A dead-lettered message is materialized into the outbox dead letter before this call. /// An exception fails the activation's messaging, which deactivates the grain. /// void OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement); diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 3ac4de6cccb..477a0f50f4b 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -67,10 +67,11 @@ an inbox sink keeps that state as its only journaled copy. Register both during construction or activation setup, before journal recovery. - `IOutbox.Register(IOutboxMessageSource)` adds a source, and - `IOutbox.SendFromSource(messageId, receiverId)` stages a reference entry with the - message ID, receiver, and delivery state, but no payload. The source provides each - envelope through `GetEnvelope`, and `OnRetired` reports - `OutboxMessageRetirement.Delivered` or `DeadLettered`. + `IOutbox.Send(messageId)` stages a reference entry with the message ID and delivery + state. The first source whose `TryGetMessage` returns the message owns it, and + returns the same `OutboxMessage` while it provides the message. A source withdraws + a pending message by returning `false`. When the message retires, `OnRetired` + reports `OutboxMessageRetirement.Delivered` or `DeadLettered`. - `IInbox.Register(IInboxMessageSink)` adds a sink. A sink whose `TryAccept` returns `true` has staged the message in its own journaled state. Sinks are idempotent. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index ed717560510..957dc4e9347 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -201,15 +201,14 @@ public readonly partial struct InboxMessage public partial interface IOutbox { IOutbox Register(IOutboxMessageSource source); + void Send(HierarchicalKey messageId); void Send(OutboxMessage message); - void SendFromSource(HierarchicalKey messageId, Runtime.GrainId receiverId); } public partial interface IOutboxMessageSource { - bool CanProvide(HierarchicalKey messageId); - Envelope GetEnvelope(HierarchicalKey messageId); void OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement); + bool TryGetMessage(HierarchicalKey messageId, out OutboxMessage message); } public sealed partial class MessageType diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs index 05a7e3d53cb..b207fde837f 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs @@ -569,7 +569,7 @@ public void Send(OutboxMessage message) _messages.Add(message); } - public void SendFromSource(HierarchicalKey messageId, GrainId receiverId) => throw new NotSupportedException(); + public void Send(HierarchicalKey messageId) => throw new NotSupportedException(); public IOutbox Register(IOutboxMessageSource source) => throw new NotSupportedException(); From 13566f465390f23c9dda4c43bc21e6ac610658f7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 12:29:18 -0700 Subject: [PATCH 14/95] feat(messaging): identify messages with MessageId wrapping HierarchicalKey MessageId is a thin readonly struct around HierarchicalKey with its own codec, copier, and JSON converter. Envelope.MessageId, IOutbox.Send, IOutboxMessageSource, MessageType, and OutboxExtensions use MessageId for message identity; HierarchicalKey is unchanged. --- src/Orleans.Messaging/Envelope.cs | 2 +- src/Orleans.Messaging/EnvelopeValidation.cs | 2 +- src/Orleans.Messaging/IOutbox.cs | 4 +- src/Orleans.Messaging/IOutboxMessageSource.cs | 8 +- src/Orleans.Messaging/MessageId.cs | 122 +++++++++++++++ src/Orleans.Messaging/MessageIdCodec.cs | 48 ++++++ src/Orleans.Messaging/MessageType.cs | 2 +- src/Orleans.Messaging/OutboxExtensions.cs | 10 +- src/Orleans.Messaging/README.md | 14 +- .../Orleans.Messaging/Orleans.Messaging.cs | 67 +++++++- .../Contracts/EnvelopeContractTests.cs | 10 +- .../Contracts/HandlerRoutingContractTests.cs | 2 +- .../Contracts/MessageIdTests.cs | 148 ++++++++++++++++++ .../Contracts/TypedMessageHelperTests.cs | 28 ++-- .../Contracts/TypedOutboxExtensionsTests.cs | 54 +++---- 15 files changed, 446 insertions(+), 75 deletions(-) create mode 100644 src/Orleans.Messaging/MessageId.cs create mode 100644 src/Orleans.Messaging/MessageIdCodec.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs index f1b29de853e..b14436531cf 100644 --- a/src/Orleans.Messaging/Envelope.cs +++ b/src/Orleans.Messaging/Envelope.cs @@ -16,7 +16,7 @@ public readonly struct Envelope { /// Gets the exact application-defined command identity within the receiving inbox. [Id(0)] - public required HierarchicalKey MessageId { get; init; } + public required MessageId MessageId { get; init; } /// Gets the required application payload bytes, including an empty payload. [Id(1)] diff --git a/src/Orleans.Messaging/EnvelopeValidation.cs b/src/Orleans.Messaging/EnvelopeValidation.cs index e3689b96190..05cc40b29d5 100644 --- a/src/Orleans.Messaging/EnvelopeValidation.cs +++ b/src/Orleans.Messaging/EnvelopeValidation.cs @@ -33,7 +33,7 @@ public static void Validate(OutboxMessage message) Validate(message.Envelope); } - internal static void ValidateMessageId(HierarchicalKey messageId) + internal static void ValidateMessageId(MessageId messageId) { if (messageId.IsDefault) throw new ArgumentException("The message ID must not be unset.", nameof(messageId)); if (Utf8.GetByteCount(messageId.ToString()) > MaxMessageIdBytes) diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index f25ac27864e..27e0ef127e2 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -54,9 +54,9 @@ public interface IOutbox /// The ID is pending as an ordinary message, no registered source provides it, the source returned a different ID, /// or the outbox is unavailable. /// - void Send(HierarchicalKey messageId); + void Send(MessageId messageId); - /// Registers a source which provides messages sent through . + /// Registers a source which provides messages sent through . /// The source. /// This outbox. /// diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index 53abb656441..87984224aeb 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -5,7 +5,7 @@ namespace Orleans.Messaging; /// /// /// -/// stages a reference entry which carries the message ID and delivery +/// stages a reference entry which carries the message ID and delivery /// state, but no payload or receiver, so the payload is journaled once, in the application's state. The outbox keeps /// every delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, /// dead-lettering after MaxDeliveryAttempts or MaxOutboxRetryAge, wake-ups, and metrics. @@ -41,11 +41,11 @@ public interface IOutboxMessageSource /// message or has withdrawn it. /// /// - /// The outbox validates the message. During , an invalid message fails the + /// The outbox validates the message. During , an invalid message fails the /// send. Afterwards, an exception or an invalid message breaks the source's contract and fails the activation's /// messaging, which deactivates the grain. /// - bool TryGetMessage(HierarchicalKey messageId, out OutboxMessage message); + bool TryGetMessage(MessageId messageId, out OutboxMessage message); /// Notifies the source that the outbox retired a message, so the source can release its payload. /// The ID of the retired reference entry. @@ -55,7 +55,7 @@ public interface IOutboxMessageSource /// A dead-lettered message is materialized into the outbox dead letter before this call. /// An exception fails the activation's messaging, which deactivates the grain. /// - void OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement); + void OnRetired(MessageId messageId, OutboxMessageRetirement retirement); } /// Describes how an outbox message retired. diff --git a/src/Orleans.Messaging/MessageId.cs b/src/Orleans.Messaging/MessageId.cs new file mode 100644 index 00000000000..c72dd23b41c --- /dev/null +++ b/src/Orleans.Messaging/MessageId.cs @@ -0,0 +1,122 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Orleans.Messaging; + +/// +/// Identifies one message by an application-supplied . +/// +/// +/// The receiver deduplicates messages by this identity, so applications reuse it when retrying a send. +/// Equality, hashing, parsing, and formatting use the key's canonical escaped path. +/// +[Immutable, Alias("Orleans.Messaging.MessageId")] +[JsonConverter(typeof(MessageIdJsonConverter))] +public readonly struct MessageId : ISpanFormattable, IEquatable, IParsable, ISpanParsable +{ + /// Creates a message identity from a hierarchical key. + /// The application identity of the message. + public MessageId(HierarchicalKey key) => Key = key; + + /// Gets the hierarchical key which identifies the message. + public HierarchicalKey Key { get; } + + /// Gets whether this value is unset. + public bool IsDefault => Key.IsDefault; + + /// Creates a message identity from literal segments in root-first order. + /// The nonempty literal segments. + /// The message identity. + /// A segment is null. + /// The input or a segment is empty. + public static MessageId Create(params ReadOnlySpan segments) => new(HierarchicalKey.Create(segments)); + + /// + public static MessageId Parse(string s, IFormatProvider? provider = null) => new(HierarchicalKey.Parse(s, provider)); + + /// + public static MessageId Parse(ReadOnlySpan s, IFormatProvider? provider = null) => new(HierarchicalKey.Parse(s, provider)); + + /// + public static bool TryParse([NotNullWhen(true)] string? s, IFormatProvider? provider, out MessageId result) + { + var parsed = HierarchicalKey.TryParse(s, provider, out var key); + result = new(key); + return parsed; + } + + /// + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out MessageId result) + { + var parsed = HierarchicalKey.TryParse(s, provider, out var key); + result = new(key); + return parsed; + } + + /// + public bool Equals(MessageId other) => Key.Equals(other.Key); + + /// + public override bool Equals(object? obj) => obj is MessageId other && Equals(other); + + /// + public override int GetHashCode() => Key.GetHashCode(); + + /// Compares message identities. + /// The first identity. + /// The second identity. + /// Whether the identities are equal. + public static bool operator ==(MessageId left, MessageId right) => left.Equals(right); + + /// Compares message identities for inequality. + /// The first identity. + /// The second identity. + /// Whether the identities differ. + public static bool operator !=(MessageId left, MessageId right) => !left.Equals(right); + + /// + public override string ToString() => Key.ToString(); + + /// + public string ToString(string? format, IFormatProvider? formatProvider) => Key.ToString(); + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) => + Key.TryFormat(destination, out charsWritten, format, provider); + + /// Serializes message identities as canonical escaped JSON strings. + public sealed class MessageIdJsonConverter : JsonConverter + { + /// Creates a message identity JSON converter. + public MessageIdJsonConverter() { } + + /// + public override MessageId Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) return default; + if (reader.TokenType is JsonTokenType.String or JsonTokenType.PropertyName + && TryParse(reader.GetString(), null, out var result)) return result; + throw new JsonException("A message identity must be a canonical escaped path."); + } + + /// + public override void Write(Utf8JsonWriter writer, MessageId value, JsonSerializerOptions options) + { + if (value.IsDefault) writer.WriteNullValue(); + else writer.WriteStringValue(value.ToString()); + } + + /// + public override MessageId ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + Read(ref reader, typeToConvert, options); + + /// + public override void WriteAsPropertyName(Utf8JsonWriter writer, MessageId value, JsonSerializerOptions options) + { + if (value.IsDefault) throw new JsonException("A message identity dictionary key must be set."); + writer.WritePropertyName(value.ToString()); + } + } +} diff --git a/src/Orleans.Messaging/MessageIdCodec.cs b/src/Orleans.Messaging/MessageIdCodec.cs new file mode 100644 index 00000000000..1a04a7b6704 --- /dev/null +++ b/src/Orleans.Messaging/MessageIdCodec.cs @@ -0,0 +1,48 @@ +using System; +using System.Buffers; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Cloning; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging; + +[RegisterSerializer] +internal sealed class MessageIdCodec(IFieldCodec strings) : IFieldCodec +{ + public void WriteField(ref Writer writer, uint fieldIdDelta, + Type? expectedType, MessageId value) where TBufferWriter : IBufferWriter + { + ReferenceCodec.MarkValueField(writer.Session); + writer.WriteFieldHeader(fieldIdDelta, expectedType, typeof(MessageId), WireType.TagDelimited); + strings.WriteField(ref writer, 0, typeof(string), value.IsDefault ? null! : value.ToString()); + writer.WriteEndObject(); + } + + public MessageId ReadValue(ref Reader reader, Field field) + { + field.EnsureWireTypeTagDelimited(); + ReferenceCodec.MarkValueField(reader.Session); + MessageId result = default; + uint fieldId = 0; + while (true) + { + var header = reader.ReadFieldHeader(); + if (header.IsEndBaseOrEndObject) return result; + fieldId += header.FieldIdDelta; + if (fieldId == 0) + { + var canonical = strings.ReadValue(ref reader, header); + result = canonical is null ? default : MessageId.Parse(canonical); + } + else reader.ConsumeUnknownField(header); + } + } +} + +[RegisterCopier] +internal sealed class MessageIdCopier : IDeepCopier +{ + public MessageId DeepCopy(MessageId input, CopyContext context) => input; +} diff --git a/src/Orleans.Messaging/MessageType.cs b/src/Orleans.Messaging/MessageType.cs index 614d1378aba..2ed2313a99b 100644 --- a/src/Orleans.Messaging/MessageType.cs +++ b/src/Orleans.Messaging/MessageType.cs @@ -86,7 +86,7 @@ public T Decode(Envelope envelope) /// /// Serialization completes before publication. Keep the published payload bytes unchanged. /// - public OutboxMessage Create(HierarchicalKey messageId, GrainId receiverId, T body, DateTimeOffset? deliverAt = null) + public OutboxMessage Create(MessageId messageId, GrainId receiverId, T body, DateTimeOffset? deliverAt = null) { if (body is null) throw new ArgumentNullException(nameof(body)); if (receiverId.IsDefault) diff --git a/src/Orleans.Messaging/OutboxExtensions.cs b/src/Orleans.Messaging/OutboxExtensions.cs index 16b4a39b786..26326440549 100644 --- a/src/Orleans.Messaging/OutboxExtensions.cs +++ b/src/Orleans.Messaging/OutboxExtensions.cs @@ -25,7 +25,7 @@ public static class OutboxExtensions /// public static void Send( this IOutbox outbox, MessageType messageType, - HierarchicalKey messageId, GrainId destination, T body, DateTimeOffset? deliverAt = null) + MessageId messageId, GrainId destination, T body, DateTimeOffset? deliverAt = null) { ArgumentNullException.ThrowIfNull(outbox); ArgumentNullException.ThrowIfNull(messageType); @@ -46,7 +46,7 @@ public static void Send( /// public static void Send( this IOutbox outbox, MessageType messageType, - HierarchicalKey messageId, IAddressable destination, T body, DateTimeOffset? deliverAt = null) + MessageId messageId, IAddressable destination, T body, DateTimeOffset? deliverAt = null) { ArgumentNullException.ThrowIfNull(destination); outbox.Send(messageType, messageId, destination.GetGrainId(), body, deliverAt); @@ -66,7 +66,7 @@ public static void Send( /// public static void Send( this IOutbox outbox, MessageType messageType, - IEnumerable<(HierarchicalKey MessageId, GrainId Destination, T Body)> messages) + IEnumerable<(MessageId MessageId, GrainId Destination, T Body)> messages) { ArgumentNullException.ThrowIfNull(outbox); ArgumentNullException.ThrowIfNull(messageType); @@ -84,13 +84,13 @@ public static void Send( /// The current received message and completion context. /// The application-selected reply destination. /// The reply body. - /// The reply identity is context.Message.Envelope.MessageId.CreateChildKey("result"). + /// The reply identity is context.Message.Envelope.MessageId.Key.CreateChildKey("result"). public static void SendReply( this IOutbox outbox, MessageType messageType, IInboxHandlerContext context, GrainId destination, T body) { ArgumentNullException.ThrowIfNull(context); - outbox.Send(messageType, context.Message.Envelope.MessageId.CreateChildKey("result"), destination, body); + outbox.Send(messageType, new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("result")), destination, body); } /// Encodes and stages a deterministic result reply to a grain reference. diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 477a0f50f4b..ee3373d2034 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -9,8 +9,8 @@ typed message bindings, delivery results, and options. `Envelope` carries: -- `MessageId`: a `HierarchicalKey` supplied by the application, at most 1,024 UTF-8 - bytes. It identifies one command at the receiver. +- `MessageId`: an application-supplied `HierarchicalKey` wrapped in a `MessageId`, at + most 1,024 UTF-8 bytes. It identifies one command at the receiver. - `Payload`: opaque bytes; an empty payload is valid. - `Subject`: an optional ordinal name, at most 256 UTF-8 bytes, that selects a typed handler. @@ -19,10 +19,12 @@ typed message bindings, delivery results, and options. `OutboxMessage` pairs an envelope with its receiver. `InboxMessage` carries the same envelope at the receiver. Keep published payload and header arrays unchanged. -`HierarchicalKey` wraps its canonical escaped path. Equality and hashing compare that -path ordinally. Use `Create`, `CreateChildKey`, and `Append` to build keys, and -`GetParent`, `IsParentOf`, `IsChildOf`, `IsAncestorOf`, and the segment enumerator to -navigate them. +`MessageId` wraps a `HierarchicalKey`. Create one with `new MessageId(key)` or +`MessageId.Create(segments)`, and read the key through `Key`. `HierarchicalKey` wraps +its canonical escaped path, and equality and hashing of both types compare that path +ordinally. Use `HierarchicalKey.Create`, `CreateChildKey`, and `Append` to build keys, +and `GetParent`, `IsParentOf`, `IsChildOf`, `IsAncestorOf`, and the segment enumerator +to navigate them. ## Handle messages diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 957dc4e9347..f1691524bc2 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -62,7 +62,7 @@ public readonly partial struct Envelope public System.Collections.Generic.Dictionary? Headers { get { throw null; } init { } } [Id(0)] - public required HierarchicalKey MessageId { get { throw null; } init { } } + public required MessageId MessageId { get { throw null; } init { } } [Id(1)] public required byte[] Payload { get { throw null; } init { } } @@ -201,14 +201,65 @@ public readonly partial struct InboxMessage public partial interface IOutbox { IOutbox Register(IOutboxMessageSource source); - void Send(HierarchicalKey messageId); + void Send(MessageId messageId); void Send(OutboxMessage message); } public partial interface IOutboxMessageSource { - void OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement); - bool TryGetMessage(HierarchicalKey messageId, out OutboxMessage message); + void OnRetired(MessageId messageId, OutboxMessageRetirement retirement); + bool TryGetMessage(MessageId messageId, out OutboxMessage message); + } + + [Immutable] + [Alias("Orleans.Messaging.MessageId")] + [System.Text.Json.Serialization.JsonConverter(typeof(MessageIdJsonConverter))] + public readonly partial struct MessageId : System.ISpanFormattable, System.IFormattable, System.IEquatable, System.IParsable, System.ISpanParsable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public MessageId(HierarchicalKey key) { } + + public bool IsDefault { get { throw null; } } + + public HierarchicalKey Key { get { throw null; } } + + public static MessageId Create(scoped params System.ReadOnlySpan segments) { throw null; } + + public readonly bool Equals(MessageId other) { throw null; } + + public override readonly bool Equals(object? obj) { throw null; } + + public override readonly int GetHashCode() { throw null; } + + public static bool operator ==(MessageId left, MessageId right) { throw null; } + + public static bool operator !=(MessageId left, MessageId right) { throw null; } + + static MessageId System.ISpanParsable.Parse(System.ReadOnlySpan s, System.IFormatProvider? provider) { throw null; } + + static MessageId System.IParsable.Parse(string s, System.IFormatProvider? provider) { throw null; } + + public override readonly string ToString() { throw null; } + + public readonly string ToString(string? format, System.IFormatProvider? formatProvider) { throw null; } + + public readonly bool TryFormat(System.Span destination, out int charsWritten, System.ReadOnlySpan format, System.IFormatProvider? provider) { throw null; } + + static bool System.ISpanParsable.TryParse(System.ReadOnlySpan s, System.IFormatProvider? provider, out MessageId result) { throw null; } + + static bool System.IParsable.TryParse(string? s, System.IFormatProvider? provider, out MessageId result) { throw null; } + + public sealed partial class MessageIdJsonConverter : System.Text.Json.Serialization.JsonConverter + { + public override MessageId Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override MessageId ReadAsPropertyName(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options) { throw null; } + + public override void Write(System.Text.Json.Utf8JsonWriter writer, MessageId value, System.Text.Json.JsonSerializerOptions options) { } + + public override void WriteAsPropertyName(System.Text.Json.Utf8JsonWriter writer, MessageId value, System.Text.Json.JsonSerializerOptions options) { } + } } public sealed partial class MessageType @@ -219,7 +270,7 @@ public MessageType(string subject, Serialization.Serializer serializer) { } public string Subject { get { throw null; } } - public OutboxMessage Create(HierarchicalKey messageId, Runtime.GrainId receiverId, T body, System.DateTimeOffset? deliverAt = null) { throw null; } + public OutboxMessage Create(MessageId messageId, Runtime.GrainId receiverId, T body, System.DateTimeOffset? deliverAt = null) { throw null; } public T Decode(Envelope envelope) { throw null; } @@ -230,11 +281,11 @@ public MessageType(string subject, Serialization.Serializer serializer) { } public static partial class OutboxExtensions { - public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.GrainId destination, T body, System.DateTimeOffset? deliverAt = null) { } + public static void Send(this IOutbox outbox, MessageType messageType, MessageId messageId, Runtime.GrainId destination, T body, System.DateTimeOffset? deliverAt = null) { } - public static void Send(this IOutbox outbox, MessageType messageType, HierarchicalKey messageId, Runtime.IAddressable destination, T body, System.DateTimeOffset? deliverAt = null) { } + public static void Send(this IOutbox outbox, MessageType messageType, MessageId messageId, Runtime.IAddressable destination, T body, System.DateTimeOffset? deliverAt = null) { } - public static void Send(this IOutbox outbox, MessageType messageType, System.Collections.Generic.IEnumerable<(HierarchicalKey MessageId, Runtime.GrainId Destination, T Body)> messages) { } + public static void Send(this IOutbox outbox, MessageType messageType, System.Collections.Generic.IEnumerable<(MessageId MessageId, Runtime.GrainId Destination, T Body)> messages) { } public static void SendReply(this IOutbox outbox, MessageType messageType, IInboxHandlerContext context, Runtime.GrainId destination, T body) { } diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 18338088159..8578f1ccbb2 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -12,7 +12,7 @@ namespace Orleans.Messaging.Tests.Contracts; [TestArea("Messaging")] public sealed class EnvelopeContractTests { - private static HierarchicalKey Id => HierarchicalKey.Create("orders", "123"); + private static MessageId Id => MessageId.Create("orders", "123"); private static GrainId Receiver => GrainId.Create("receiver", "123"); [Fact] @@ -253,7 +253,7 @@ public void PendingCommandEqualityIgnoresExtraHeadersAndSender(string change, bo ReceiverId = change == "receiver" ? GrainId.Create("receiver", "other") : Receiver, Envelope = envelope with { - MessageId = change == "id" ? HierarchicalKey.Create("other") : change == "child" ? Id.CreateChildKey("child") : Id, + MessageId = change == "id" ? MessageId.Create("other") : change == "child" ? new MessageId(Id.Key.CreateChildKey("child")) : Id, Payload = change == "payload" ? [0, 255, 129] : change == "payload-length" ? [0, 255] : [0, 255, 128], Subject = change == "subject" ? "other" : change == "subject-case" ? "Subject.v1" : change == "no-subject" ? null : "subject.v1", Headers = new() { ["custom"] = Encoding.UTF8.GetBytes(change == "headers" ? "second" : "first"), ["sender"] = change == "sender" ? [2] : [1] }, @@ -301,9 +301,9 @@ public void IdentityAndSubjectAdmissionBoundaries(string field, int length, bool { MessageId = field switch { - "id" => HierarchicalKey.Create(new string('x', length)), - "depth" => HierarchicalKey.Create(Enumerable.Repeat("x", length).ToArray()), - "unicode-id" => HierarchicalKey.Create(new string('\u00e9', length)), + "id" => MessageId.Create(new string('x', length)), + "depth" => MessageId.Create(Enumerable.Repeat("x", length).ToArray()), + "unicode-id" => MessageId.Create(new string('\u00e9', length)), _ => Id }, Subject = field.Contains("subject", StringComparison.Ordinal) ? new string(field == "subject" ? 's' : '\u00e9', length) : null diff --git a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs index f81df0f7e25..b87b63e618d 100644 --- a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -87,7 +87,7 @@ private MessageType CreateType(string subject) => private static FakeContext CreateContext(MessageType type, string body) => new(new InboxMessage { - Envelope = type.Create(HierarchicalKey.Create(Guid.NewGuid().ToString("N")), GrainId.Create("test", "receiver"), body).Envelope + Envelope = type.Create(MessageId.Create(Guid.NewGuid().ToString("N")), GrainId.Create("test", "receiver"), body).Envelope }); private sealed class FakeContext(InboxMessage message) : IInboxHandlerContext diff --git a/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs new file mode 100644 index 00000000000..fa53e6dc82e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs @@ -0,0 +1,148 @@ +using System.Buffers; +using System.Reflection; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class MessageIdTests +{ + [Fact] + public void SerializationContract_IdentifiesCurrentType() + { + var type = typeof(MessageId); + Assert.Equal("Orleans.Messaging.MessageId", Assert.Single(type.GetCustomAttributes()).Alias); + Assert.True(type.IsValueType); + Assert.Empty(type.GetCustomAttributes()); + } + + [Fact] + public void Identity_WrapsKeyAndComparesByCanonicalPath() + { + var key = HierarchicalKey.Create("orders", "a/b", "reserve"); + var id = new MessageId(key); + + Assert.Equal(key, id.Key); + Assert.False(id.IsDefault); + Assert.Equal(id, MessageId.Create("orders", "a/b", "reserve")); + Assert.True(id == new MessageId(HierarchicalKey.Parse(key.ToString()))); + Assert.True(id != MessageId.Create("orders", "a/b")); + Assert.Equal(key.GetHashCode(), id.GetHashCode()); + Assert.False(id.Equals((object)key)); + Assert.True(default(MessageId).IsDefault); + Assert.Equal(default, new MessageId(default)); + } + + [Fact] + public void ParseAndFormat_UseCanonicalEscapedPath() + { + var id = MessageId.Create("orders", "a/b", @"c\d"); + const string canonical = @"orders/a\/b/c\\d"; + + Assert.Equal(canonical, id.ToString()); + Assert.Equal(canonical, $"{id}"); + Assert.Equal(id, MessageId.Parse(canonical)); + Assert.Equal(id, MessageId.Parse(canonical.AsSpan())); + Assert.True(MessageId.TryParse(canonical, null, out var parsed)); + Assert.Equal(id, parsed); + Assert.True(MessageId.TryParse(canonical.AsSpan(), null, out parsed)); + Assert.Equal(id, parsed); + var buffer = new char[canonical.Length]; + Assert.True(id.TryFormat(buffer, out var written, default, null)); + Assert.Equal(canonical, new string(buffer, 0, written)); + Assert.False(id.TryFormat(buffer.AsSpan(0, canonical.Length - 1), out written, default, null)); + Assert.Equal(0, written); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Parse_InvalidPath_IsRejected(string text) + { + Assert.False(MessageId.TryParse(text, null, out var result)); + Assert.True(result.IsDefault); + Assert.False(MessageId.TryParse(text.AsSpan(), null, out result)); + Assert.True(result.IsDefault); + Assert.Throws(() => MessageId.Parse(text)); + Assert.False(MessageId.TryParse(null, null, out result)); + Assert.True(result.IsDefault); + } + + [Fact] + public void Json_ValueAndDictionaryKeyRoundTripCanonicalEscapedPath() + { + var id = MessageId.Create("orders", "a/b", @"c\d"); + + var json = JsonSerializer.Serialize(id); + Assert.Equal(JsonSerializer.Serialize(@"orders/a\/b/c\\d"), json); + Assert.Equal(id, JsonSerializer.Deserialize(json)); + + var map = JsonSerializer.Serialize(new Dictionary { [id] = 42 }); + Assert.Equal(JsonSerializer.Serialize(new Dictionary { [id.ToString()] = 42 }), map); + Assert.Equal(42, JsonSerializer.Deserialize>(map)![id]); + } + + [Fact] + public void Json_DefaultValueRoundTripsAsNullAndIsRejectedAsDictionaryKey() + { + Assert.Equal("null", JsonSerializer.Serialize(default(MessageId))); + Assert.True(JsonSerializer.Deserialize("null").IsDefault); + Assert.Throws(() => JsonSerializer.Serialize(new Dictionary { [default] = 1 })); + } + + [Theory] + [InlineData("\"\"")] + [InlineData("\"a//b\"")] + [InlineData("42")] + public void Json_MalformedValueIsRejected(string json) => + Assert.Throws(() => JsonSerializer.Deserialize(json)); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SerializationAndCopy_PreserveIdentity(bool unset) + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var id = unset ? default : MessageId.Create("tenant", @"a\/b", "42"); + var serializer = first.GetRequiredService>(); + var bytes = serializer.SerializeToArray(id); + var decoded = second.GetRequiredService>().Deserialize(bytes); + + Assert.Equal(id, decoded); + Assert.Equal(id.IsDefault, decoded.IsDefault); + Assert.Equal(id, first.GetRequiredService().Copy(id)); + Assert.Equal(bytes, serializer.SerializeToArray(decoded)); + var map = first.GetRequiredService>>(); + var dictionary = Assert.IsType>(map.Deserialize(map.SerializeToArray(new() { [id] = 42 }))); + Assert.Equal(42, dictionary[decoded]); + } + + [Theory] + [InlineData("")] + [InlineData("a//b")] + [InlineData(@"a\q")] + public void Codec_InvalidCanonicalWire_IsRejected(string canonical) + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var session = services.GetRequiredService().GetSession(); + var output = new ArrayBufferWriter(); + var writer = Writer.Create(output, session); + ReferenceCodec.MarkValueField(session); + writer.WriteFieldHeader(0, typeof(MessageId), typeof(MessageId), WireType.TagDelimited); + services.GetRequiredService>().WriteField(ref writer, 0, typeof(string), canonical); + writer.WriteEndObject(); + writer.Commit(); + Assert.Throws(() => services.GetRequiredService>().Deserialize(output.WrittenSpan)); + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs index 21d3103f220..dafa286824d 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs @@ -23,7 +23,7 @@ public sealed class TypedMessageHelperTests private const string Subject = "orders.reserve"; private static GrainId Sender => GrainId.Create("sender", "typed-helper"); private static GrainId Receiver => GrainId.Create("receiver", "warehouse"); - private static HierarchicalKey MessageKey => HierarchicalKey.Create("orders", "42", "reserve"); + private static MessageId MessageKey => MessageId.Create("orders", "42", "reserve"); [Fact] public void MessageType_RejectsNullSerializer() @@ -345,8 +345,8 @@ public void MessageType_CreateRejectsInvalidIdentitiesBeforeCodecWrite(string in var key = invalidIdentity switch { "messageId" => default, - "key-ascii-bytes" => HierarchicalKey.Create(new string('k', 1025)), - "key-multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', 341) + "ab"), + "key-ascii-bytes" => MessageId.Create(new string('k', 1025)), + "key-multibyte-bytes" => MessageId.Create(new string('\u20ac', 341) + "ab"), _ => MessageKey }; OutboxMessage? result = null; @@ -369,7 +369,7 @@ public void MessageType_CreateUsesExplicitIdentitiesAndSuppliedHierarchicalKey() using var services = CreateServices(); var serializer = services.GetRequiredService>(); var messageType = new MessageType(Subject, serializer); - var key = HierarchicalKey.Create("orders", "42", "reserve"); + var key = MessageId.Create("orders", "42", "reserve"); var first = messageType.Create(key, Receiver, "reserve first"); var otherReceiver = GrainId.Create("receiver", "other-warehouse"); var second = messageType.Create(key, otherReceiver, "reserve second"); @@ -378,8 +378,8 @@ public void MessageType_CreateUsesExplicitIdentitiesAndSuppliedHierarchicalKey() AssertEnvelopeIdentity(second, key, otherReceiver, Subject); Assert.Equal("orders/42/reserve", first.Envelope.MessageId.ToString()); Assert.Equal("orders/42/reserve", second.Envelope.MessageId.ToString()); - Assert.Equal(3, first.Envelope.MessageId.CountSegments()); - Assert.Equal(3, second.Envelope.MessageId.CountSegments()); + Assert.Equal(3, first.Envelope.MessageId.Key.CountSegments()); + Assert.Equal(3, second.Envelope.MessageId.Key.CountSegments()); Assert.Equal("reserve first", serializer.Deserialize(first.Envelope.Payload)); Assert.Equal("reserve second", serializer.Deserialize(second.Envelope.Payload)); } @@ -395,9 +395,9 @@ public void MessageType_CreateAcceptsMessageKeyAtAdmissionLimits(string variatio var messageType = new MessageType(Subject, serializer); var key = variation switch { - "segments" => HierarchicalKey.Create(Enumerable.Repeat("s", 32).ToArray()), - "ascii-bytes" => HierarchicalKey.Create(new string('k', 1024)), - "multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', 341) + "a"), + "segments" => MessageId.Create(Enumerable.Repeat("s", 32).ToArray()), + "ascii-bytes" => MessageId.Create(new string('k', 1024)), + "multibyte-bytes" => MessageId.Create(new string('\u20ac', 341) + "a"), _ => throw new ArgumentOutOfRangeException(nameof(variation)) }; @@ -407,7 +407,7 @@ public void MessageType_CreateAcceptsMessageKeyAtAdmissionLimits(string variatio AssertEnvelopeIdentity(envelope, key, Receiver, Subject); if (variation == "segments") { - Assert.Equal(32, envelope.Envelope.MessageId.CountSegments()); + Assert.Equal(32, envelope.Envelope.MessageId.Key.CountSegments()); } else { @@ -534,13 +534,13 @@ public void MessageType_EnvelopeCopySharesStableArrayAcrossSubsequentCreate() Assert.Same(original.Envelope.Payload, surviving.Envelope.Payload); var expected = original.Envelope.Payload.ToArray(); original = default; - var subsequent = messageType.Create(MessageKey.CreateChildKey("next"), Receiver, "new reserve"); + var subsequent = messageType.Create(new MessageId(MessageKey.Key.CreateChildKey("next")), Receiver, "new reserve"); Assert.Equal(2, probe.WriteCount); Assert.NotSame(surviving.Envelope.Payload, subsequent.Envelope.Payload); Assert.Equal(expected, surviving.Envelope.Payload.ToArray()); Assert.Equal(MessageKey, surviving.Envelope.MessageId); - Assert.Equal(MessageKey.CreateChildKey("next"), subsequent.Envelope.MessageId); + Assert.Equal(new MessageId(MessageKey.Key.CreateChildKey("next")), subsequent.Envelope.MessageId); Assert.Equal("surviving reserve", services.GetRequiredService>().Deserialize(surviving.Envelope.Payload)); Assert.Equal("new reserve", services.GetRequiredService>().Deserialize(subsequent.Envelope.Payload)); } @@ -1765,7 +1765,7 @@ private static Envelope Envelope(string subject, byte[] payload) } private static void AssertEnvelopeIdentity( - OutboxMessage message, HierarchicalKey key, GrainId receiver, string subject) + OutboxMessage message, MessageId key, GrainId receiver, string subject) { Assert.Equal(key, message.Envelope.MessageId); Assert.Equal(receiver, message.ReceiverId); @@ -1941,7 +1941,7 @@ void Mutate(string body, IInboxHandlerContext caller) { calls++; stock--; - outbox.Send(type, MessageKey.CreateChildKey("result"), Receiver, body); + outbox.Send(type, new MessageId(MessageKey.Key.CreateChildKey("result")), Receiver, body); caller.Complete(); } if (asynchronous) diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs index b207fde837f..79d671e24dd 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs @@ -22,7 +22,7 @@ public sealed class TypedOutboxExtensionsTests private static GrainId Owner => GrainId.Create("warehouse", "reply-owner"); private static GrainId ReceivedSender => GrainId.Create("customer", "original-command-sender"); private static GrainId Destination => GrainId.Create("audit", "explicit-destination"); - private static HierarchicalKey CommandId => HierarchicalKey.Create("orders", "42", "reserve"); + private static MessageId CommandId => MessageId.Create("orders", "42", "reserve"); [Fact] public void SendBatch_EnumeratesOnceAndStagesEachMessageBeforeAdvancing() @@ -31,20 +31,20 @@ public void SendBatch_EnumeratesOnceAndStagesEachMessageBeforeAdvancing() var type = new MessageType(Subject, services.GetRequiredService>()); var outbox = new StoringOutbox(); var enumerations = 0; - IEnumerable<(HierarchicalKey, GrainId, string)> Messages() + IEnumerable<(MessageId, GrainId, string)> Messages() { Assert.Equal(1, ++enumerations); yield return (CommandId, Destination, "first"); Assert.Equal(1, outbox.RawSendCalls); Assert.Equal("first", type.Decode(Assert.Single(outbox.Messages))); - yield return (CommandId.CreateChildKey("second"), ReceivedSender, "second"); + yield return (new MessageId(CommandId.Key.CreateChildKey("second")), ReceivedSender, "second"); } outbox.Send(type, Messages()); Assert.Equal(1, enumerations); Assert.Equal(2, outbox.RawSendCalls); - Assert.Equal(new[] { CommandId, CommandId.CreateChildKey("second") }, outbox.Messages.Select(value => value.Envelope.MessageId)); + Assert.Equal(new[] { CommandId, new MessageId(CommandId.Key.CreateChildKey("second")) }, outbox.Messages.Select(value => value.Envelope.MessageId)); Assert.Equal(new[] { Destination, ReceivedSender }, outbox.Messages.Select(value => value.ReceiverId)); Assert.All(outbox.Messages, value => Assert.Equal(Subject, GetSubject(value.Envelope))); Assert.Equal(new[] { "first", "second" }, outbox.Messages.Select(type.Decode)); @@ -61,12 +61,12 @@ public void SendBatch_PreparationFailurePreservesEarlierIntentAndOriginalError(b var type = new MessageType(Subject, new Serializer(codec, sessions)); var outbox = new StoringOutbox(); var sentinel = new InvalidDataException("later preparation failed"); - IEnumerable<(HierarchicalKey, GrainId, string)> Messages() + IEnumerable<(MessageId, GrainId, string)> Messages() { yield return (CommandId, Destination, "first"); if (!encodingFailure) throw sentinel; codec.NextFailure = sentinel; - yield return (CommandId.CreateChildKey("second"), Destination, "second"); + yield return (new MessageId(CommandId.Key.CreateChildKey("second")), Destination, "second"); } Assert.Same(sentinel, Assert.Throws(() => outbox.Send(type, Messages()))); @@ -78,12 +78,12 @@ public void SendBatch_PreparationFailurePreservesEarlierIntentAndOriginalError(b var earlierPayload = earlier.Envelope.Payload.ToArray(); Assert.Equal("first", type.Decode(earlier)); Assert.Equal(encodingFailure ? 2 : 1, codec.Writes); - outbox.Send(type, CommandId.CreateChildKey("second"), Destination, "recovered"); + outbox.Send(type, new MessageId(CommandId.Key.CreateChildKey("second")), Destination, "recovered"); Assert.Equal(new[] { "first", "recovered" }, outbox.Messages.Select(type.Decode)); Assert.Equal(2, outbox.RawSendCalls); var retained = outbox.Messages.First(); Assert.Equal(earlierPayload, retained.Envelope.Payload); - Assert.Equal(new[] { CommandId, CommandId.CreateChildKey("second") }, + Assert.Equal(new[] { CommandId, new MessageId(CommandId.Key.CreateChildKey("second")) }, outbox.Messages.Select(message => message.Envelope.MessageId)); Assert.All(outbox.Messages, message => Assert.Equal(Destination, message.ReceiverId)); } @@ -98,7 +98,7 @@ public void SendBatch_RejectsNullArgumentsBeforeEncodingOrStaging(string variati var actual = new StoringOutbox(); var type = new MessageType(Subject, services.GetRequiredService>()); IOutbox outbox = actual; - IEnumerable<(HierarchicalKey, GrainId, string)> messages = [(CommandId, Destination, "body")]; + IEnumerable<(MessageId, GrainId, string)> messages = [(CommandId, Destination, "body")]; switch (variation) { case "outbox": outbox = null!; break; @@ -136,10 +136,10 @@ public void SendBatch_RawFailurePreservesPublishedArrayAndOriginalError(bool sto body[body.Length / 2] = 17; var sentinel = new IOException("raw staging failed"); var outbox = new StoringOutbox() { Failure = sentinel, StoreBeforeFailure = storeBeforeFailure }; - (HierarchicalKey, GrainId, byte[])[] messages = + (MessageId, GrainId, byte[])[] messages = [ (CommandId, Destination, body), - (CommandId.CreateChildKey("second"), Destination, body) + (new MessageId(CommandId.Key.CreateChildKey("second")), Destination, body) ]; Assert.Same(sentinel, Assert.Throws(() => outbox.Send(type, messages))); @@ -226,9 +226,9 @@ public void SendReply_UsesResultChildAndExplicitDestination(bool addressable) else outbox.SendReply(type, context, Destination, "approved €42"); var staged = Assert.Single(outbox.Messages); - Assert.Equal(CommandId.CreateChildKey("result"), staged.Envelope.MessageId); + Assert.Equal(new MessageId(CommandId.Key.CreateChildKey("result")), staged.Envelope.MessageId); Assert.Equal("orders/42/reserve/result", staged.Envelope.MessageId.ToString()); - Assert.Equal(4, staged.Envelope.MessageId.CountSegments()); + Assert.Equal(4, staged.Envelope.MessageId.Key.CountSegments()); Assert.Equal(Destination, staged.ReceiverId); Assert.NotEqual(ReceivedSender, staged.ReceiverId); Assert.Equal(Subject, GetSubject(staged.Envelope)); @@ -255,9 +255,9 @@ public void SendReply_ResultChildAtAdmissionLimitEncodesAndStages(bool addressab var type = new MessageType(Subject, serializer); var key = variation switch { - "segments" => HierarchicalKey.Create(Enumerable.Repeat("s", 31).ToArray()), - "ascii-bytes" => HierarchicalKey.Create(new string('k', 1017)), - "multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', 339)), + "segments" => MessageId.Create(Enumerable.Repeat("s", 31).ToArray()), + "ascii-bytes" => MessageId.Create(new string('k', 1017)), + "multibyte-bytes" => MessageId.Create(new string('\u20ac', 339)), _ => throw new ArgumentOutOfRangeException(nameof(variation)) }; var context = Context(key); @@ -266,8 +266,8 @@ public void SendReply_ResultChildAtAdmissionLimitEncodesAndStages(bool addressab Invoke(outbox, type, context, key, Destination, "boundary result", reply: true, addressable: addressable); var staged = Assert.Single(outbox.Messages); - Assert.Equal(key.CreateChildKey("result"), staged.Envelope.MessageId); - if (variation == "segments") Assert.Equal(32, staged.Envelope.MessageId.CountSegments()); + Assert.Equal(new MessageId(key.Key.CreateChildKey("result")), staged.Envelope.MessageId); + if (variation == "segments") Assert.Equal(32, staged.Envelope.MessageId.Key.CountSegments()); else Assert.Equal(1024, System.Text.Encoding.UTF8.GetByteCount(staged.Envelope.MessageId.ToString())); Assert.Equal(Destination, staged.ReceiverId); Assert.Equal(serializer.SerializeToArray("boundary result"), staged.Envelope.Payload.ToArray()); @@ -301,7 +301,7 @@ public void TypedSend_StoresPublishedArrayAndPreservesBytesAfterRemoval(bool rep Assert.Same(published.Envelope.Payload, outbox.WireSeen); Assert.NotSame(body, published.Envelope.Payload); Assert.Equal(body, serializer.Deserialize(published.Envelope.Payload)); - Assert.Equal(reply ? CommandId.CreateChildKey("result") : CommandId, staged.Envelope.MessageId); + Assert.Equal(reply ? new MessageId(CommandId.Key.CreateChildKey("result")) : CommandId, staged.Envelope.MessageId); Assert.Equal(Destination, staged.ReceiverId); Assert.Equal(0, context.CompletionCount); outbox.ClearMessages(); @@ -348,7 +348,7 @@ public void TypedSend_RawSendFailurePreservesExactExceptionAndPublishedArray(boo Assert.Equal(serializer.SerializeToArray(body), outbox.WireSeen.ToArray()); var published = outbox.LastPublished!.Value; Assert.Equal(Destination, published.ReceiverId); - Assert.Equal(reply ? CommandId.CreateChildKey("result") : CommandId, published.Envelope.MessageId); + Assert.Equal(reply ? new MessageId(CommandId.Key.CreateChildKey("result")) : CommandId, published.Envelope.MessageId); Assert.Equal(Subject, GetSubject(published.Envelope)); Assert.Same(outbox.WireSeen, published.Envelope.Payload); if (storeBeforeFailure) @@ -396,7 +396,7 @@ public void TypedSend_PartialEncodingFailureNeverStagesAndNextCallHasPristineWir var staged = Assert.Single(outbox.Messages); Assert.Equal(direct.SerializeToArray("recovered €42"), staged.Envelope.Payload.ToArray()); Assert.Equal("recovered €42", direct.Deserialize(staged.Envelope.Payload)); - Assert.Equal(reply ? CommandId.CreateChildKey("result") : CommandId, staged.Envelope.MessageId); + Assert.Equal(reply ? new MessageId(CommandId.Key.CreateChildKey("result")) : CommandId, staged.Envelope.MessageId); Assert.Equal(2, codec.Writes); Assert.Equal(1, outbox.RawSendCalls); Assert.Equal(0, context.CompletionCount); @@ -434,8 +434,8 @@ public void TypedSend_ValidationFailureNeverEncodesOrStages(bool reply, bool add { "messageId" or "unset-parent" => default, // Reply adds seven UTF8 bytes ("/result"). - "bytes" => HierarchicalKey.Create(new string('k', reply ? 1018 : 1025)), - "multibyte-bytes" => HierarchicalKey.Create(new string('\u20ac', reply ? 339 : 341) + (reply ? "a" : "ab")), + "bytes" => MessageId.Create(new string('k', reply ? 1018 : 1025)), + "multibyte-bytes" => MessageId.Create(new string('\u20ac', reply ? 339 : 341) + (reply ? "a" : "ab")), _ => CommandId }; var context = Context(reply ? key : null); @@ -494,7 +494,7 @@ public void TypedSend_ValidationFailureNeverEncodesOrStages(bool reply, bool add private static ServiceProvider CreateServices() => new ServiceCollection().AddSerializer().BuildServiceProvider(); - private static CountingContext Context(HierarchicalKey? key = null) + private static CountingContext Context(MessageId? key = null) { var messageId = key ?? CommandId; if (messageId == default) return new(new InboxMessage { Envelope = default }); @@ -515,7 +515,7 @@ private static CountingContext Context(HierarchicalKey? key = null) } private static void Invoke(IOutbox outbox, MessageType type, CountingContext context, - HierarchicalKey key, GrainId destination, T body, bool reply, bool addressable) + MessageId key, GrainId destination, T body, bool reply, bool addressable) { if (addressable) { @@ -569,11 +569,11 @@ public void Send(OutboxMessage message) _messages.Add(message); } - public void Send(HierarchicalKey messageId) => throw new NotSupportedException(); + public void Send(MessageId messageId) => throw new NotSupportedException(); public IOutbox Register(IOutboxMessageSource source) => throw new NotSupportedException(); - public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) + public bool TryGetMessage(MessageId messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) { foreach (var message in _messages) if (message.Envelope.MessageId == messageId) From ba272a1f520a205fb723a345a4d5cabcf1a397d0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 13:00:23 -0700 Subject: [PATCH 15/95] feat(messaging): dead-letter outbox messages whose source fails After a send, an outbox source that throws or returns an invalid message dead-letters that entry alone and receives OnRetired(DeadLettered) in the same journal write. An exception during Send propagates without staging anything. --- src/Orleans.Messaging/IOutbox.cs | 4 ++- src/Orleans.Messaging/IOutboxMessageSource.cs | 33 +++++++++++++------ src/Orleans.Messaging/README.md | 5 ++- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index 27e0ef127e2..fd6204be757 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -43,7 +43,9 @@ public interface IOutbox /// The outbox journals the message ID and delivery state, while the source holds the payload and receiver, and /// delivers the message like one staged through . Stage the message in the source's /// state before this call: the outbox resolves the source now, reads and validates its message, and fails when none - /// provides it. Sending a pending ID again coalesces with the pending message. A message which a source addresses to + /// provides it; an exception from the source propagates and nothing is staged. After the send, a source which fails + /// to provide a valid message dead-letters the entry instead; see . Sending a + /// pending ID again coalesces with the pending message. A message which a source addresses to /// the sending grain is rejected; send it through , which stages it in the grain's /// own inbox. /// diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index 87984224aeb..0e4d52f67dd 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -12,9 +12,11 @@ namespace Orleans.Messaging; /// /// /// The outbox calls when the message is sent and before every delivery attempt. The first -/// registered source which returns the message owns it for the activation's lifetime. When the entry retires, the -/// outbox calls the owner's exactly once, synchronously, while it stages the entry's removal, so -/// changes the source stages in its journaled state commit in the same journal write as that removal. +/// registered source which returns the message owns it for the activation's lifetime; for an entry recovered from the +/// journal, the first source which doesn't return , including one which throws, owns it. When the +/// entry retires, the outbox calls the owner's exactly once, synchronously, while it stages the +/// entry's removal, so changes the source stages in its journaled state commit in the same journal write as that +/// removal. /// /// /// A source withdraws a pending message by returning from , for @@ -23,6 +25,14 @@ namespace Orleans.Messaging; /// flight still completes, so a source can receive for a message it withdrew. /// /// +/// A source which can't provide a pending message after its send, for example because its stored payload no longer +/// decodes, throws from ; an invalid message is handled the same way. The outbox then +/// dead-letters that entry alone, without delivering it, and calls with +/// ; the grain's messaging and its other messages are unaffected. Because no valid message exists, the dead letter's message carries only its +/// , with an empty payload and a default , and its +/// reason records the failure. +/// +/// /// All calls run on the activation scheduler. Register sources during activation setup, before journal recovery: a /// recovered reference entry fails the activation when no source is registered. While a source provides a message, it /// returns the same message, envelope and receiver alike, for every call. @@ -41,9 +51,11 @@ public interface IOutboxMessageSource /// message or has withdrawn it. /// /// - /// The outbox validates the message. During , an invalid message fails the - /// send. Afterwards, an exception or an invalid message breaks the source's contract and fails the activation's - /// messaging, which deactivates the grain. + /// The outbox validates the message. During , an exception or an invalid + /// message fails the send, and nothing is staged. Afterwards, the entry is durable: an exception or an invalid message, + /// including one with a different or addressed to the sending grain, dead-letters the + /// entry with its failure as the reason, counting the failed materialization as a delivery attempt, and the outbox + /// then calls with . /// bool TryGetMessage(MessageId messageId, out OutboxMessage message); @@ -52,8 +64,9 @@ public interface IOutboxMessageSource /// Whether the receiver retired the message or the message was dead-lettered. /// /// Stage the release synchronously in journaled state; it commits in the journal write which removes the entry. - /// A dead-lettered message is materialized into the outbox dead letter before this call. - /// An exception fails the activation's messaging, which deactivates the grain. + /// A dead-lettered message is materialized into the outbox dead letter before this call, unless the source failed to + /// provide it. Don't depend on decoding the payload here: this call also retires a message whose payload the source + /// can no longer provide. An exception fails the activation's messaging, which deactivates the grain. /// void OnRetired(MessageId messageId, OutboxMessageRetirement retirement); } @@ -65,8 +78,8 @@ public enum OutboxMessageRetirement Delivered = 1, /// - /// The receiver dead-lettered the message, or the outbox dead-lettered it after MaxDeliveryAttempts or once - /// it was older than MaxOutboxRetryAge. + /// The receiver dead-lettered the message, or the outbox dead-lettered it after MaxDeliveryAttempts, once it + /// was older than MaxOutboxRetryAge, or because its source failed to provide it. /// DeadLettered = 2 } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index ee3373d2034..c54d369220d 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -73,7 +73,10 @@ construction or activation setup, before journal recovery. state. The first source whose `TryGetMessage` returns the message owns it, and returns the same `OutboxMessage` while it provides the message. A source withdraws a pending message by returning `false`. When the message retires, `OnRetired` - reports `OutboxMessageRetirement.Delivered` or `DeadLettered`. + reports `OutboxMessageRetirement.Delivered` or `DeadLettered`. After the send, a + source that throws or returns an invalid message dead-letters that entry alone, + with `OnRetired(DeadLettered)` in the same journal write; an exception during + `Send` propagates without staging anything. - `IInbox.Register(IInboxMessageSink)` adds a sink. A sink whose `TryAccept` returns `true` has staged the message in its own journaled state. Sinks are idempotent. From d4965bd78d898b904b67c12f7f628ece0749dd80 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 15:16:27 -0700 Subject: [PATCH 16/95] feat(messaging): offer scheduled messages to inbox sinks and let sinks report delivery results IInboxMessageSink.TryAccept reports a DeliveryResult: Accepted, Duplicate, Backpressured, or DeadLettered. A sink that throws or reports another status dead-letters the message. Sinks are also offered messages with DeliverAt and honor it for the messages they take. A Backpressured result for a self-sent message fails the send. --- src/Orleans.Messaging/IInboxMessageSink.cs | 67 +++++++++++++------ src/Orleans.Messaging/IOutbox.cs | 3 +- src/Orleans.Messaging/README.md | 12 +++- .../Orleans.Messaging/Orleans.Messaging.cs | 2 +- 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/src/Orleans.Messaging/IInboxMessageSink.cs b/src/Orleans.Messaging/IInboxMessageSink.cs index cb8f9667dbe..02fce8bb42e 100644 --- a/src/Orleans.Messaging/IInboxMessageSink.cs +++ b/src/Orleans.Messaging/IInboxMessageSink.cs @@ -5,36 +5,65 @@ namespace Orleans.Messaging; /// /// /// -/// After validation and deduplication, the inbox offers each message without to the -/// registered sinks in registration order, including messages the grain sends to itself. A sink which returns -/// has synchronously staged the message's effects in its own journaled state. The inbox then -/// journals only the message's processed record, without its payload, in the same journal write, and reports -/// once that write is durable. Run effects which depend on the write, such as -/// starting work, from Orleans.Journaling.IStateMachine.OnWriteCompleted. +/// After validation and deduplication, the inbox offers each message to the registered sinks in registration order, +/// including messages the grain sends to itself and messages with . The first sink +/// which takes the message decides its . A sink which reports +/// has synchronously staged the message's effects in its own journaled state. The +/// inbox then journals only the message's processed record, without its payload, in the same journal write, and reports +/// the result once that write is durable. Run effects which depend on the write, such as starting work, from +/// Orleans.Journaling.IStateMachine.OnWriteCompleted. /// /// -/// A sink-accepted message never occupies inbox capacity and has no inbox retry or dead-letter lifecycle: the sink -/// owns its processing. Messages which every sink declines, and messages with , are -/// accepted by the inbox and dispatched to instances as usual. +/// A sink-owned message never occupies inbox capacity and has no inbox retry lifecycle: the sink owns its processing, +/// including honoring the message's . Messages which every sink declines are accepted by +/// the inbox and dispatched to instances once their has +/// passed; decline a scheduled message to have the inbox hold it until then. /// /// -/// Sinks must be idempotent. The inbox deduplicates the message ID only while its processed record is retained for -/// DeduplicationWindow; a later redelivery is offered again. A message whose journal write fails is -/// redelivered to the next activation, which recovers the sink's state from the journal. +/// Sinks must be idempotent. The inbox deduplicates the message ID while its processed record is retained for +/// DeduplicationWindow; after that, the message is offered again, and a sink which still holds it reports +/// . A message whose journal write fails is redelivered to the next activation, +/// which recovers the sink's state from the journal. /// /// public interface IInboxMessageSink { - /// Accepts a message into the sink's journaled state, or declines it. + /// Takes a message into the sink's journaled state, or declines it. /// The validated, deduplicated message. + /// + /// When this method returns , the delivery result which the inbox reports to the sender. + /// /// - /// when the sink staged the message's effects; when it declines the - /// message without staging anything. + /// when the sink takes the message and reports ; + /// when it declines the message without staging anything. /// /// - /// Runs synchronously on the activation scheduler. Validate the message before staging any change. A sink which - /// throws must leave no staged change: the inbox dead-letters the message with its payload, records its ID as - /// processed, and reports . + /// + /// Runs synchronously on the activation scheduler. Validate the message before staging any change. The sink reports + /// one of these statuses: + /// + /// + /// + /// : the sink staged the message's effects, and the inbox records the message + /// as processed in the same write. + /// + /// + /// : the sink already holds the message and stages nothing. The inbox reports + /// it once any write in progress, which can hold the original, is durable. + /// + /// + /// : the sink is at capacity and stages nothing, so the sender retries + /// later. A message the grain sends to itself can't wait for a retry, so this status fails that send. + /// + /// + /// : the sink rejects the message and stages nothing. The inbox + /// dead-letters it with its payload and the result's reason, and records it as processed. + /// + /// + /// + /// A sink which throws or reports another status must leave no staged change, and the inbox dead-letters the message + /// with the failure as its reason. + /// /// - bool TryAccept(InboxMessage message); + bool TryAccept(InboxMessage message, out DeliveryResult result); } diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index fd6204be757..b9f74136b54 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -30,7 +30,8 @@ public interface IOutbox /// /// The envelope has an unset identity, invalid subject or destination, or exceeds identity/subject limits. /// - /// The identity conflicts, or the outbox is unavailable. + /// The identity conflicts, an inbox sink backpressured a message addressed to the sending grain, or the outbox is + /// unavailable. /// void Send(OutboxMessage message); diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index c54d369220d..c79c4ee29bd 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -77,8 +77,16 @@ construction or activation setup, before journal recovery. source that throws or returns an invalid message dead-letters that entry alone, with `OnRetired(DeadLettered)` in the same journal write; an exception during `Send` propagates without staging anything. -- `IInbox.Register(IInboxMessageSink)` adds a sink. A sink whose `TryAccept` returns - `true` has staged the message in its own journaled state. Sinks are idempotent. +- `IInbox.Register(IInboxMessageSink)` adds a sink. The inbox offers each message, + including self-sent and scheduled messages, to the sinks in order. The first sink + whose `TryAccept` returns `true` reports the `DeliveryResult`: + - `Accepted`: the sink staged the message in its own journaled state. + - `Duplicate`: the sink already holds the message. + - `Backpressured`: the sink is at capacity. A self-sent message fails its `Send`. + - `DeadLettered`: the sink rejects the message with a reason. A sink that throws + or reports another status dead-letters the message. + + A sink that takes a scheduled message honors its `DeliverAt`. Sinks are idempotent. ## Delivery results diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index f1691524bc2..9de59c3a61a 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -176,7 +176,7 @@ public partial interface IInboxHandlerContext public partial interface IInboxMessageSink { - bool TryAccept(InboxMessage message); + bool TryAccept(InboxMessage message, out DeliveryResult result); } public static partial class InboxExtensions From c1a05214fbad8e652c0357aaf3b723f3886d664c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 15:25:00 -0700 Subject: [PATCH 17/95] refactor(messaging)!: decode message types from envelopes only MessageType.Decode accepts an Envelope. Callers pass InboxMessage.Envelope or OutboxMessage.Envelope. BREAKING CHANGE: MessageType.Decode(InboxMessage) and Decode(OutboxMessage) are removed. --- src/Orleans.Messaging/InboxExtensions.cs | 2 +- src/Orleans.Messaging/MessageType.cs | 10 ------ .../Orleans.Messaging/Orleans.Messaging.cs | 4 --- .../Contracts/EnvelopeContractTests.cs | 2 +- .../Contracts/TypedMessageHelperTests.cs | 34 +++++++------------ .../Contracts/TypedOutboxExtensionsTests.cs | 10 +++--- 6 files changed, 19 insertions(+), 43 deletions(-) diff --git a/src/Orleans.Messaging/InboxExtensions.cs b/src/Orleans.Messaging/InboxExtensions.cs index 580fddf18ca..ac67ff879c8 100644 --- a/src/Orleans.Messaging/InboxExtensions.cs +++ b/src/Orleans.Messaging/InboxExtensions.cs @@ -83,7 +83,7 @@ public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken can { cancellationToken.ThrowIfCancellationRequested(); // The delegate owns the decoded body, including when cancellation arrives during decoding. - var body = messageType.Decode(context.Message); + var body = messageType.Decode(context.Message.Envelope); return Invoke(body, context, cancellationToken); } diff --git a/src/Orleans.Messaging/MessageType.cs b/src/Orleans.Messaging/MessageType.cs index 2ed2313a99b..c8fb9914671 100644 --- a/src/Orleans.Messaging/MessageType.cs +++ b/src/Orleans.Messaging/MessageType.cs @@ -64,16 +64,6 @@ public T Decode(Envelope envelope) ?? throw new ArgumentException($"Message subject '{Subject}' requires a nonnull payload.", nameof(envelope)); } - /// Decodes the payload of a received message. - /// The received message. - /// The nonnull decoded payload. - public T Decode(InboxMessage message) => Decode(message.Envelope); - - /// Decodes the payload of an outgoing message. - /// The outgoing message. - /// The nonnull decoded payload. - public T Decode(OutboxMessage message) => Decode(message.Envelope); - /// Encodes a typed body into an outgoing message. /// The stable application command identity. /// The destination inbox. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 9de59c3a61a..4b9a22579c2 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -273,10 +273,6 @@ public MessageType(string subject, Serialization.Serializer serializer) { } public OutboxMessage Create(MessageId messageId, Runtime.GrainId receiverId, T body, System.DateTimeOffset? deliverAt = null) { throw null; } public T Decode(Envelope envelope) { throw null; } - - public T Decode(InboxMessage message) { throw null; } - - public T Decode(OutboxMessage message) { throw null; } } public static partial class OutboxExtensions diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 8578f1ccbb2..1c6d2f2f309 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -148,7 +148,7 @@ public void DeliverAtRoundTripsThroughCreateAndIndependentWire() Assert.Equal(deliverAt, actual.Envelope.DeliverAt); Assert.Equal(deliverAt.Offset, actual.Envelope.DeliverAt!.Value.Offset); Assert.Equal(Receiver, actual.ReceiverId); - Assert.Equal("scheduled", type.Decode(actual)); + Assert.Equal("scheduled", type.Decode(actual.Envelope)); Assert.True(Invoke("EnvelopeEquivalence", "AreEquivalent", outgoing, actual)); } } diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs index dafa286824d..42f475ed70a 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedMessageHelperTests.cs @@ -47,7 +47,7 @@ public void MessageType_CreatedFromSerializer_RoundTripsBody() Assert.Equal(Subject, messageType.Subject); Assert.Equal(Subject, message.Envelope.Subject); - Assert.Equal("payload", messageType.Decode(message)); + Assert.Equal("payload", messageType.Decode(message.Envelope)); } [Fact] @@ -577,7 +577,7 @@ public void MessageType_CreateLeavesResourceBodyAndDecodedValueApplicationOwned( Assert.Equal(0, body.DisposeCount); Assert.Equal(MessageKey, retained.Envelope.MessageId); Assert.Equal(services.GetRequiredService>().SerializeToArray(body.Name), retained.Envelope.Payload.ToArray()); - using var decoded = type.Decode(retained); + using var decoded = type.Decode(retained.Envelope); Assert.NotSame(body, decoded); Assert.Same(codec.LastDecoded, decoded); Assert.Equal("resource €42", decoded.Name); @@ -1623,8 +1623,8 @@ private static void AssertClosedSerializerFactory(T body) var envelope = first.Create(MessageKey, Receiver, body); var other = second.Create(MessageKey, Receiver, body); Assert.Equal(serializer.SerializeToArray(body), envelope.Envelope.Payload.ToArray()); - Assert.Equal(body, first.Decode(envelope)); - Assert.Equal(body, second.Decode(other)); + Assert.Equal(body, first.Decode(envelope.Envelope)); + Assert.Equal(body, second.Decode(other.Envelope)); AssertEnvelopeIdentity(envelope, MessageKey, Receiver, Subject); AssertEnvelopeIdentity(other, MessageKey, Receiver, "orders.cancel"); Assert.NotSame(first, second); @@ -1662,8 +1662,8 @@ private static void AssertCustomClosedSerializerPreserved(T body, bool factor var other = second.Create(MessageKey, Receiver, body); Assert.Equal(2, probe.WriteCount); Assert.Equal(codecServices.GetRequiredService>().SerializeToArray(body), envelope.Envelope.Payload.ToArray()); - Assert.Equal(body, first.Decode(envelope)); - Assert.Equal(body, second.Decode(other)); + Assert.Equal(body, first.Decode(envelope.Envelope)); + Assert.Equal(body, second.Decode(other.Envelope)); Assert.Equal(2, probe.ReadCount); Assert.Equal(factory ? 1 : 0, factoryCalls); AssertEnvelopeIdentity(envelope, MessageKey, Receiver, Subject); @@ -1894,17 +1894,8 @@ public void MessageType_DecodeRejectsAbsentSubjectBeforeCodecRead() probe.ReadFailure = new InvalidDataException("payload codec must not run"); var type = new MessageType(Subject, serializer); var envelope = new Envelope { MessageId = MessageKey, Payload = direct.SerializeToArray("unrouted €42") }; - var outgoing = new OutboxMessage { Envelope = envelope, ReceiverId = Receiver }; - foreach (var decode in new Func[] - { - () => type.Decode(envelope), - () => type.Decode(outgoing), - () => type.Decode(outgoing.ToInboxMessage()) - }) - { - Assert.Equal("envelope", Assert.Throws(() => decode()).ParamName); - } + Assert.Equal("envelope", Assert.Throws(() => type.Decode(envelope)).ParamName); Assert.Equal(0, probe.ReadCount); Assert.Equal(0, probe.WriteCount); Assert.Equal("unrouted €42", direct.Deserialize(envelope.Payload)); @@ -2022,8 +2013,7 @@ public void MessageType_CreateSharesEnvelopeObjectsAndDecodesAllMessageRoles() Assert.Same(envelope.Subject, incoming.Envelope.Subject); Assert.Null(incoming.Envelope.Headers); Assert.Equal("shared payload € / 東京", type.Decode(envelope)); - Assert.Equal("shared payload € / 東京", type.Decode(outgoing)); - Assert.Equal("shared payload € / 東京", type.Decode(incoming)); + Assert.Equal("shared payload € / 東京", type.Decode(incoming.Envelope)); } [Theory] @@ -2051,10 +2041,10 @@ public void InboxDispatcher_SubjectHeaderLookupDoesNotAllocateDecodedSubjectStri var context = new CountingInboxHandlerContext(outgoing); var state = new HandlerState(); var dispatcher = new TestInbox().Register(type, state, ObserveState); - for (var i = 0; i < 64; i++) type.Decode(outgoing); + for (var i = 0; i < 64; i++) type.Decode(outgoing.Envelope); MeasureDispatchAllocations(dispatcher, context, 64); var before = GC.GetAllocatedBytesForCurrentThread(); - for (var i = 0; i < 256; i++) type.Decode(outgoing); + for (var i = 0; i < 256; i++) type.Decode(outgoing.Envelope); var decodeBytes = GC.GetAllocatedBytesForCurrentThread() - before; var dispatchBytes = MeasureDispatchAllocations(dispatcher, context, 256); @@ -2067,7 +2057,7 @@ public void InboxDispatcher_SubjectHeaderLookupDoesNotAllocateDecodedSubjectStri Assert.Equal(0, context.CompletionCount); Assert.Equal(SubjectAtLimit(multibyte: true), GetSubject(outgoing.Envelope)); Assert.Equal(MessageKey, outgoing.Envelope.MessageId); - Assert.Equal("allocation control", type.Decode(context.Message)); + Assert.Equal("allocation control", type.Decode(context.Message.Envelope)); } [Fact] @@ -2157,7 +2147,7 @@ public void AddMessageType_GrainIdPayloadCollapsesSharedSerializerRegistration() Assert.Equal(2, probe.WriteCount); Assert.Equal(Receiver, outgoing.ReceiverId); Assert.Equal(MessageKey, outgoing.Envelope.MessageId); - Assert.Equal(Receiver, customBinding.Decode(outgoing)); + Assert.Equal(Receiver, customBinding.Decode(outgoing.Envelope)); Assert.Null(outgoing.Envelope.Headers); Assert.Equal(2, probe.ReadCount); Assert.Equal(1, factoryCalls); diff --git a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs index 79d671e24dd..c2cd0a2828f 100644 --- a/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/TypedOutboxExtensionsTests.cs @@ -36,7 +36,7 @@ public void SendBatch_EnumeratesOnceAndStagesEachMessageBeforeAdvancing() Assert.Equal(1, ++enumerations); yield return (CommandId, Destination, "first"); Assert.Equal(1, outbox.RawSendCalls); - Assert.Equal("first", type.Decode(Assert.Single(outbox.Messages))); + Assert.Equal("first", type.Decode(Assert.Single(outbox.Messages).Envelope)); yield return (new MessageId(CommandId.Key.CreateChildKey("second")), ReceivedSender, "second"); } @@ -47,7 +47,7 @@ public void SendBatch_EnumeratesOnceAndStagesEachMessageBeforeAdvancing() Assert.Equal(new[] { CommandId, new MessageId(CommandId.Key.CreateChildKey("second")) }, outbox.Messages.Select(value => value.Envelope.MessageId)); Assert.Equal(new[] { Destination, ReceivedSender }, outbox.Messages.Select(value => value.ReceiverId)); Assert.All(outbox.Messages, value => Assert.Equal(Subject, GetSubject(value.Envelope))); - Assert.Equal(new[] { "first", "second" }, outbox.Messages.Select(type.Decode)); + Assert.Equal(new[] { "first", "second" }, outbox.Messages.Select(message => type.Decode(message.Envelope))); } [Theory] @@ -76,10 +76,10 @@ public void SendBatch_PreparationFailurePreservesEarlierIntentAndOriginalError(b Assert.Equal(Destination, earlier.ReceiverId); Assert.Equal(Subject, GetSubject(earlier.Envelope)); var earlierPayload = earlier.Envelope.Payload.ToArray(); - Assert.Equal("first", type.Decode(earlier)); + Assert.Equal("first", type.Decode(earlier.Envelope)); Assert.Equal(encodingFailure ? 2 : 1, codec.Writes); outbox.Send(type, new MessageId(CommandId.Key.CreateChildKey("second")), Destination, "recovered"); - Assert.Equal(new[] { "first", "recovered" }, outbox.Messages.Select(type.Decode)); + Assert.Equal(new[] { "first", "recovered" }, outbox.Messages.Select(message => type.Decode(message.Envelope))); Assert.Equal(2, outbox.RawSendCalls); var retained = outbox.Messages.First(); Assert.Equal(earlierPayload, retained.Envelope.Payload); @@ -203,7 +203,7 @@ public void Send_CarriesOptionalDeliverAt(bool addressable) Assert.Equal(deliverAt, staged.Envelope.DeliverAt); Assert.Equal(deliverAt.Offset, staged.Envelope.DeliverAt!.Value.Offset); Assert.Equal(Destination, staged.ReceiverId); - Assert.Equal("scheduled", type.Decode(staged)); + Assert.Equal("scheduled", type.Decode(staged.Envelope)); Assert.Equal(deliverAt, type.Create(CommandId, Destination, "scheduled", deliverAt).Envelope.DeliverAt); Assert.Null(type.Create(CommandId, Destination, "scheduled").Envelope.DeliverAt); } From 50c7f653ba13a911b2c33cbb89fbee5e1132b9c6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 15:30:08 -0700 Subject: [PATCH 18/95] refactor(messaging): rename IInboxExtension to IMessagingExtension The grain extension becomes IMessagingExtension with alias IMessagingExtension, so later messaging operations can join it without another extension interface. --- ...boxExtension.cs => IMessagingExtension.cs} | 6 ++-- src/Orleans.Messaging/OrleansContracts.txt | 2 +- .../Orleans.Messaging/Orleans.Messaging.cs | 36 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) rename src/Orleans.Messaging/{IInboxExtension.cs => IMessagingExtension.cs} (88%) diff --git a/src/Orleans.Messaging/IInboxExtension.cs b/src/Orleans.Messaging/IMessagingExtension.cs similarity index 88% rename from src/Orleans.Messaging/IInboxExtension.cs rename to src/Orleans.Messaging/IMessagingExtension.cs index 0d83a287783..d3fc72c603d 100644 --- a/src/Orleans.Messaging/IInboxExtension.cs +++ b/src/Orleans.Messaging/IMessagingExtension.cs @@ -8,10 +8,10 @@ namespace Orleans.Messaging; /// -/// Non-generic grain extension for inbox message delivery. +/// The grain extension through which other grains exchange durable messages with this grain. /// -[Alias("IInboxExtension")] -public interface IInboxExtension : IGrainExtension +[Alias("IMessagingExtension")] +public interface IMessagingExtension : IGrainExtension { /// /// Delivers a message to this grain's inbox. diff --git a/src/Orleans.Messaging/OrleansContracts.txt b/src/Orleans.Messaging/OrleansContracts.txt index 4a090bf615d..71a144be71d 100644 --- a/src/Orleans.Messaging/OrleansContracts.txt +++ b/src/Orleans.Messaging/OrleansContracts.txt @@ -10,5 +10,5 @@ # Review every diff: identity or signature changes can break wire compatibility during rolling upgrades. # Details: https://aka.ms/orleans/OrleansContracts.txt -interface [GrainInterfaceType("Orleans.Messaging.IInboxExtension")] Orleans.Messaging.IInboxExtension [Version(0)] +interface [GrainInterfaceType("Orleans.Messaging.IMessagingExtension")] Orleans.Messaging.IMessagingExtension [Version(0)] DeliverAsync: DeliverAsync(Orleans.Messaging.InboxMessage, System.Threading.CancellationToken) -> ValueTask diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 4b9a22579c2..79225cb5d37 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -153,13 +153,6 @@ public partial interface IInbox IInbox Register(IInboxMessageSink sink); } - [Alias("IInboxExtension")] - public partial interface IInboxExtension : Runtime.IGrainExtension, Runtime.IAddressable - { - [Alias("DeliverAsync")] - System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); - } - public partial interface IInboxHandler { bool CanHandle(IInboxHandlerContext context); @@ -179,6 +172,13 @@ public partial interface IInboxMessageSink bool TryAccept(InboxMessage message, out DeliveryResult result); } + [Alias("IMessagingExtension")] + public partial interface IMessagingExtension : Runtime.IGrainExtension, Runtime.IAddressable + { + [Alias("DeliverAsync")] + System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); + } + public static partial class InboxExtensions { public static IInbox Register(this IInbox inbox, MessageType messageType, System.Action handler) { throw null; } @@ -395,18 +395,18 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] - public sealed partial class Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + public sealed partial class Codec_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec { - public Codec_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + public Codec_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } - public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) { } + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 instance) { } - public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + public Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } - public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 instance) + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 instance) where TBufferWriter : System.Buffers.IBufferWriter { } - public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 value) + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 value) where TBufferWriter : System.Buffers.IBufferWriter { } } @@ -451,11 +451,11 @@ public Copier_InboxMessage(global::Orleans.Serialization.Activators.IActivator, global::Orleans.Serialization.Cloning.IDeepCopier + public sealed partial class Copier_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 : global::Orleans.Serialization.Cloning.IDeepCopier, global::Orleans.Serialization.Cloning.IDeepCopier { - public Copier_Invokable_IInboxExtension_GrainReference_Ext_5F40CF46(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + public Copier_Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } - public Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 DeepCopy(Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + public Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 DeepCopy(Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } } [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] @@ -471,8 +471,8 @@ public Copier_OutboxMessage(global::Orleans.Serialization.Activators.IActivator< [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] - [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.Messaging.IInboxExtension), typeof(global::Orleans.Messaging.IInboxExtension), "DeliverAsync" })] - public sealed partial class Invokable_IInboxExtension_GrainReference_Ext_5F40CF46 : global::Orleans.Runtime.Request + [global::Orleans.CompoundTypeAlias(new[] { "inv", typeof(global::Orleans.Runtime.GrainReference), "Ext", typeof(global::Orleans.Messaging.IMessagingExtension), typeof(global::Orleans.Messaging.IMessagingExtension), "DeliverAsync" })] + public sealed partial class Invokable_IMessagingExtension_GrainReference_Ext_93AB2A04 : global::Orleans.Runtime.Request { public global::Orleans.Messaging.InboxMessage arg0; public System.Threading.CancellationToken arg1; From 77995436dc662384fdeda4747a99eaf0510ca87e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 16:12:58 -0700 Subject: [PATCH 19/95] feat(messaging): run outbox delivery attempts through a Polly resilience pipeline DurableMessagingOptions.DeliveryResiliencePipeline supplies the Polly pipeline which runs each outbox delivery attempt. Exceptions which leave the pipeline retry durably until MaxOutboxRetryAge without consuming MaxDeliveryAttempts. --- .../Configuration/DurableMessagingOptions.cs | 32 ++++++++++++++++++- .../Orleans.Messaging.csproj | 1 + .../Orleans.Messaging/Orleans.Messaging.cs | 2 ++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index f2d5b6b43ba..f8bd6f97bcf 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -1,4 +1,5 @@ using System; +using Polly; namespace Orleans.Messaging.Configuration; @@ -102,8 +103,12 @@ public class DurableMessagingOptions public int MaxProcessingAttempts { get; set; } = 5; /// - /// Gets or sets the maximum number of attempts before an outbox message is dead-lettered. + /// Gets or sets the maximum number of delivery attempts which receivers answer without taking an outbox message + /// before the message is dead-lettered. /// + /// + /// Attempts which fail with an exception retry until elapses. + /// public int MaxDeliveryAttempts { get; set; } = 100; /// @@ -131,6 +136,31 @@ public class DurableMessagingOptions /// public int OutboxBatchSize { get; set; } = 32; + /// + /// Gets or sets the resilience pipeline which runs each outbox delivery attempt. + /// + /// + /// + /// The outbox calls each receiver through this pipeline, so its strategies, such as timeouts, retries, and circuit + /// breakers, act within one delivery attempt. When an exception leaves the pipeline, the outbox retries the + /// delivery durably after the maximum backoff until + /// elapses, so an infrastructure outage delays messages rather than + /// dead-lettering them. Receiving inboxes deduplicate by message ID, so retrying a delivery whose outcome is + /// unknown is safe. + /// + /// + /// One pipeline instance serves every outbox delivery in the silo, so a stateful strategy such as a circuit + /// breaker observes correlated failures across all receivers. + /// + /// + /// When this property is , messaging uses a default pipeline built from the silo's + /// response timeout, timed by the durable jobs clock. The default retries a failed call five times with jittered exponential + /// backoff from one second, limits the attempt to six response timeouts, and opens a circuit breaker for one + /// response timeout when at least half of 20 or more calls within 30 seconds fail. + /// calls the receiver directly. + /// + /// + public ResiliencePipeline? DeliveryResiliencePipeline { get; set; } /// /// Validates the configuration values and throws if any are invalid. diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj index 599c7880a8a..dd2b37b2bbf 100644 --- a/src/Orleans.Messaging/Orleans.Messaging.csproj +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -18,6 +18,7 @@ + diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 79225cb5d37..d0bc28bf138 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -318,6 +318,8 @@ public partial class DurableMessagingOptions public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } + public Polly.ResiliencePipeline? DeliveryResiliencePipeline { get { throw null; } set { } } + public int InboxBatchSize { get { throw null; } set { } } public int MaxCapacity { get { throw null; } set { } } From 061ac5ee752da6bbf07085866087d9480c623d0b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 17:50:16 -0700 Subject: [PATCH 20/95] feat(messaging)!: demote failed deliveries to later batches and retry backpressure indefinitely Remove the default delivery resilience pipeline and its shared circuit breaker; DeliveryResiliencePipeline defaults to ResiliencePipeline.Empty. Failed attempts reschedule durably with exponential backoff while other messages continue. Backpressured retries until the receiver takes the message. HandlerNotFound logs an error and dead-letters after MaxHandlerNotFoundRetryAge (6 hours). Unanswered attempts dead-letter after MaxOutboxRetryAge, which restarts on each receiver answer. MaxDeliveryAttempts is removed. --- .../Configuration/DurableMessagingOptions.cs | 74 ++++++++++++------- src/Orleans.Messaging/IOutboxMessageSource.cs | 6 +- .../Orleans.Messaging/Orleans.Messaging.cs | 4 +- .../DeliveryAndOptionsContractTests.cs | 15 +++- 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index f8bd6f97bcf..a17d13eb09f 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -79,8 +79,9 @@ public class DurableMessagingOptions /// /// /// - /// The outbox applies exponential backoff from this duration when delivery fails or the target inbox - /// returns DeliveryResult.Backpressured(). The inbox applies the same backoff after a handler fails. + /// The outbox applies exponential backoff from this duration after each failed delivery attempt, including when + /// the target inbox returns DeliveryResult.Backpressured(). The inbox applies the same backoff after a + /// handler fails. /// /// /// A shorter delay (e.g., 100ms) enables faster recovery when the target processes messages quickly, @@ -103,18 +104,34 @@ public class DurableMessagingOptions public int MaxProcessingAttempts { get; set; } = 5; /// - /// Gets or sets the maximum number of delivery attempts which receivers answer without taking an outbox message - /// before the message is dead-lettered. + /// Gets or sets how long the outbox retries a message whose delivery attempts end without an answer from the + /// receiver, such as during an infrastructure outage, before the message is dead-lettered. /// /// - /// Attempts which fail with an exception retry until elapses. + /// + /// The period starts with the first unanswered attempt and restarts whenever the receiver answers. A receiver + /// checks its deduplication records before answering, so only unanswered attempts can have been accepted, and this + /// period stays within to keep their retries deduplicated. + /// + /// + /// The outbox retries a message which the receiver answers with DeliveryResult.Backpressured() until the + /// receiver takes it. + /// /// - public int MaxDeliveryAttempts { get; set; } = 100; + /// Must be greater than zero and less than . Defaults to 1 day. + public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromDays(1); /// - /// Gets or sets the maximum age of an outbox message. + /// Gets or sets how long the outbox retries a message whose receiver reports no inbox handler, before the message + /// is dead-lettered. /// - public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromDays(1); + /// + /// The period starts when the receiver first answers with DeliveryResult.HandlerNotFound(), which gives a + /// deployment time to register the handler, and restarts whenever the receiver answers otherwise. The outbox logs + /// each such answer as an error. + /// + /// Must be greater than zero. Defaults to 6 hours. + public TimeSpan MaxHandlerNotFoundRetryAge { get; set; } = TimeSpan.FromHours(6); /// /// Gets or sets how long inbox and outbox dead letters are retained. @@ -142,25 +159,17 @@ public class DurableMessagingOptions /// /// /// The outbox calls each receiver through this pipeline, so its strategies, such as timeouts, retries, and circuit - /// breakers, act within one delivery attempt. When an exception leaves the pipeline, the outbox retries the - /// delivery durably after the maximum backoff until - /// elapses, so an infrastructure outage delays messages rather than - /// dead-lettering them. Receiving inboxes deduplicate by message ID, so retrying a delivery whose outcome is - /// unknown is safe. - /// - /// - /// One pipeline instance serves every outbox delivery in the silo, so a stateful strategy such as a circuit - /// breaker observes correlated failures across all receivers. + /// breakers, act within one delivery attempt. When an attempt fails, the outbox schedules the next attempt with + /// backoff and continues delivering other messages. Receiving inboxes + /// deduplicate by message ID, so retrying a delivery whose outcome is unknown is safe. /// /// - /// When this property is , messaging uses a default pipeline built from the silo's - /// response timeout, timed by the durable jobs clock. The default retries a failed call five times with jittered exponential - /// backoff from one second, limits the attempt to six response timeouts, and opens a circuit breaker for one - /// response timeout when at least half of 20 or more calls within 30 seconds fail. - /// calls the receiver directly. + /// The outbox treats a receiver's answer as proof that no earlier call for the message is still in flight, so the + /// pipeline makes one receiver call at a time. One pipeline instance serves every outbox delivery in the silo. /// /// - public ResiliencePipeline? DeliveryResiliencePipeline { get; set; } + /// Defaults to , which calls the receiver directly. + public ResiliencePipeline DeliveryResiliencePipeline { get; set; } = ResiliencePipeline.Empty; /// /// Validates the configuration values and throws if any are invalid. @@ -170,6 +179,7 @@ public class DurableMessagingOptions /// or if is less than or equal to , /// or if a retry, retention, or batch option is outside its supported range. /// + /// Thrown if is . /// /// This method is typically called by the dependency injection container during service registration /// to ensure configuration values are valid before the system starts. @@ -204,11 +214,6 @@ public void Validate() throw new ArgumentOutOfRangeException(nameof(MaxProcessingAttempts), MaxProcessingAttempts, "MaxProcessingAttempts must be greater than zero."); } - if (MaxDeliveryAttempts <= 0) - { - throw new ArgumentOutOfRangeException(nameof(MaxDeliveryAttempts), MaxDeliveryAttempts, "MaxDeliveryAttempts must be greater than zero."); - } - if (MaxOutboxRetryAge <= TimeSpan.Zero) { throw new ArgumentOutOfRangeException(nameof(MaxOutboxRetryAge), MaxOutboxRetryAge, "MaxOutboxRetryAge must be greater than TimeSpan.Zero."); @@ -222,6 +227,19 @@ public void Validate() "MaxOutboxRetryAge must be less than DeduplicationWindow."); } + if (MaxHandlerNotFoundRetryAge <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(MaxHandlerNotFoundRetryAge), + MaxHandlerNotFoundRetryAge, + "MaxHandlerNotFoundRetryAge must be greater than TimeSpan.Zero."); + } + + if (DeliveryResiliencePipeline is null) + { + throw new ArgumentNullException(nameof(DeliveryResiliencePipeline)); + } + if (DeadLetterRetentionPeriod <= TimeSpan.Zero) { throw new ArgumentOutOfRangeException( diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index 0e4d52f67dd..c9c6ce8a339 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -8,7 +8,7 @@ namespace Orleans.Messaging; /// stages a reference entry which carries the message ID and delivery /// state, but no payload or receiver, so the payload is journaled once, in the application's state. The outbox keeps /// every delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, -/// dead-lettering after MaxDeliveryAttempts or MaxOutboxRetryAge, wake-ups, and metrics. +/// dead-lettering after MaxHandlerNotFoundRetryAge or MaxOutboxRetryAge, wake-ups, and metrics. /// /// /// The outbox calls when the message is sent and before every delivery attempt. The first @@ -78,8 +78,8 @@ public enum OutboxMessageRetirement Delivered = 1, /// - /// The receiver dead-lettered the message, or the outbox dead-lettered it after MaxDeliveryAttempts, once it - /// was older than MaxOutboxRetryAge, or because its source failed to provide it. + /// The receiver dead-lettered the message, or the outbox dead-lettered it after its retry limits elapsed or because + /// its source failed to provide it. /// DeadLettered = 2 } diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index d0bc28bf138..770415df43e 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -318,13 +318,13 @@ public partial class DurableMessagingOptions public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } - public Polly.ResiliencePipeline? DeliveryResiliencePipeline { get { throw null; } set { } } + public Polly.ResiliencePipeline DeliveryResiliencePipeline { get { throw null; } set { } } public int InboxBatchSize { get { throw null; } set { } } public int MaxCapacity { get { throw null; } set { } } - public int MaxDeliveryAttempts { get { throw null; } set { } } + public System.TimeSpan MaxHandlerNotFoundRetryAge { get { throw null; } set { } } public System.TimeSpan MaxOutboxRetryAge { get { throw null; } set { } } diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index 8c3b4a6ad60..ebb193bf9ed 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -1,4 +1,5 @@ using Orleans.Messaging.Configuration; +using Polly; using Xunit; namespace Orleans.Messaging.Tests.Contracts; @@ -51,7 +52,8 @@ public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() Assert.Equal(TimeSpan.FromDays(1), options.MaxOutboxRetryAge); Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); Assert.Equal(5, options.MaxProcessingAttempts); - Assert.Equal(100, options.MaxDeliveryAttempts); + Assert.Equal(TimeSpan.FromHours(6), options.MaxHandlerNotFoundRetryAge); + Assert.Same(ResiliencePipeline.Empty, options.DeliveryResiliencePipeline); Assert.Equal(1000, options.MaxRetainedDeadLetters); Assert.Equal(32, options.InboxBatchSize); Assert.Equal(32, options.OutboxBatchSize); @@ -67,7 +69,7 @@ public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContrac (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), (nameof(DurableMessagingOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), - (nameof(DurableMessagingOptions.MaxDeliveryAttempts), options => options.MaxDeliveryAttempts = 0), + (nameof(DurableMessagingOptions.MaxHandlerNotFoundRetryAge), options => options.MaxHandlerNotFoundRetryAge = TimeSpan.Zero), (nameof(DurableMessagingOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), (nameof(DurableMessagingOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), (nameof(DurableMessagingOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), @@ -85,6 +87,15 @@ public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContrac } + [Fact] + public void Validate_NullDeliveryResiliencePipeline_Fails() + { + var options = new DurableMessagingOptions { DeliveryResiliencePipeline = null! }; + + var exception = Assert.Throws(options.Validate); + Assert.Equal(nameof(DurableMessagingOptions.DeliveryResiliencePipeline), exception.ParamName); + } + [Fact] public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() { From 133564c1585654e76d2e5d07105f6b5b27ac5421 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 19:10:24 -0700 Subject: [PATCH 21/95] refactor(messaging)!: track one outbox retry period and drop MaxHandlerNotFoundRetryAge OutboxMessageState records a single RetryingSince, which starts with the first failed attempt and restarts on each Backpressured answer. MaxOutboxRetryAge (now 6 hours) bounds both unanswered attempts and HandlerNotFound answers, and dead letters keep the last failure as their reason. --- .../Configuration/DurableMessagingOptions.cs | 38 +++++-------------- src/Orleans.Messaging/IOutboxMessageSource.cs | 2 +- .../Orleans.Messaging/Orleans.Messaging.cs | 2 - .../DeliveryAndOptionsContractTests.cs | 4 +- 4 files changed, 11 insertions(+), 35 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index a17d13eb09f..08654faedb7 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -104,34 +104,22 @@ public class DurableMessagingOptions public int MaxProcessingAttempts { get; set; } = 5; /// - /// Gets or sets how long the outbox retries a message whose delivery attempts end without an answer from the - /// receiver, such as during an infrastructure outage, before the message is dead-lettered. + /// Gets or sets how long the outbox retries a message whose delivery attempts fail, such as during an + /// infrastructure outage or while the receiver reports no inbox handler, before the message is dead-lettered. /// /// /// - /// The period starts with the first unanswered attempt and restarts whenever the receiver answers. A receiver - /// checks its deduplication records before answering, so only unanswered attempts can have been accepted, and this - /// period stays within to keep their retries deduplicated. + /// The period starts with the first failed attempt and restarts whenever the receiver answers with + /// DeliveryResult.Backpressured(), so the outbox retries a backpressured message until the receiver takes it. + /// A receiver checks its deduplication records before answering, so only attempts since the period started can have + /// been accepted, and the period stays within to keep their retries deduplicated. /// /// - /// The outbox retries a message which the receiver answers with DeliveryResult.Backpressured() until the - /// receiver takes it. + /// The outbox logs each DeliveryResult.HandlerNotFound() answer as an error. /// /// - /// Must be greater than zero and less than . Defaults to 1 day. - public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromDays(1); - - /// - /// Gets or sets how long the outbox retries a message whose receiver reports no inbox handler, before the message - /// is dead-lettered. - /// - /// - /// The period starts when the receiver first answers with DeliveryResult.HandlerNotFound(), which gives a - /// deployment time to register the handler, and restarts whenever the receiver answers otherwise. The outbox logs - /// each such answer as an error. - /// - /// Must be greater than zero. Defaults to 6 hours. - public TimeSpan MaxHandlerNotFoundRetryAge { get; set; } = TimeSpan.FromHours(6); + /// Must be greater than zero and less than . Defaults to 6 hours. + public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromHours(6); /// /// Gets or sets how long inbox and outbox dead letters are retained. @@ -227,14 +215,6 @@ public void Validate() "MaxOutboxRetryAge must be less than DeduplicationWindow."); } - if (MaxHandlerNotFoundRetryAge <= TimeSpan.Zero) - { - throw new ArgumentOutOfRangeException( - nameof(MaxHandlerNotFoundRetryAge), - MaxHandlerNotFoundRetryAge, - "MaxHandlerNotFoundRetryAge must be greater than TimeSpan.Zero."); - } - if (DeliveryResiliencePipeline is null) { throw new ArgumentNullException(nameof(DeliveryResiliencePipeline)); diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index c9c6ce8a339..c4d1970987e 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -8,7 +8,7 @@ namespace Orleans.Messaging; /// stages a reference entry which carries the message ID and delivery /// state, but no payload or receiver, so the payload is journaled once, in the application's state. The outbox keeps /// every delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, -/// dead-lettering after MaxHandlerNotFoundRetryAge or MaxOutboxRetryAge, wake-ups, and metrics. +/// dead-lettering after MaxOutboxRetryAge, wake-ups, and metrics. /// /// /// The outbox calls when the message is sent and before every delivery attempt. The first diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 770415df43e..72039e2f5bc 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -324,8 +324,6 @@ public partial class DurableMessagingOptions public int MaxCapacity { get { throw null; } set { } } - public System.TimeSpan MaxHandlerNotFoundRetryAge { get { throw null; } set { } } - public System.TimeSpan MaxOutboxRetryAge { get { throw null; } set { } } public int MaxProcessingAttempts { get { throw null; } set { } } diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index ebb193bf9ed..702cc5961fd 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -49,10 +49,9 @@ public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() Assert.Equal(1000, options.MaxCapacity); Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); - Assert.Equal(TimeSpan.FromDays(1), options.MaxOutboxRetryAge); + Assert.Equal(TimeSpan.FromHours(6), options.MaxOutboxRetryAge); Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); Assert.Equal(5, options.MaxProcessingAttempts); - Assert.Equal(TimeSpan.FromHours(6), options.MaxHandlerNotFoundRetryAge); Assert.Same(ResiliencePipeline.Empty, options.DeliveryResiliencePipeline); Assert.Equal(1000, options.MaxRetainedDeadLetters); Assert.Equal(32, options.InboxBatchSize); @@ -69,7 +68,6 @@ public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContrac (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), (nameof(DurableMessagingOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), - (nameof(DurableMessagingOptions.MaxHandlerNotFoundRetryAge), options => options.MaxHandlerNotFoundRetryAge = TimeSpan.Zero), (nameof(DurableMessagingOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), (nameof(DurableMessagingOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), (nameof(DurableMessagingOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), From a717d909f39a17383cd773e586ab0f63536da26d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 08:30:33 -0700 Subject: [PATCH 22/95] refactor(messaging)!: align outbox retry terminology with Temporal Rename the retry options and state after Temporal's retry policy: - DurableMessagingOptions.BackpressureRetryDelay becomes RetryInitialInterval. - DurableMessagingOptions.MaxOutboxRetryAge becomes OutboxRetryTimeout. - OutboxMessageState.RetryingSince becomes ExpirationTime and stores the deadline: the first failed attempt plus OutboxRetryTimeout. A Backpressured answer clears it, and the next failed attempt sets a new deadline. BREAKING CHANGE: BackpressureRetryDelay and MaxOutboxRetryAge are renamed to RetryInitialInterval and OutboxRetryTimeout. --- .../Configuration/DurableMessagingOptions.cs | 44 +++++++++---------- src/Orleans.Messaging/IOutboxMessageSource.cs | 2 +- .../Orleans.Messaging/Orleans.Messaging.cs | 8 ++-- .../DeliveryAndOptionsContractTests.cs | 16 +++---- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index 08654faedb7..35dbe4ec2c6 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -75,12 +75,12 @@ public class DurableMessagingOptions public TimeSpan DeduplicationWindow { get; set; } = TimeSpan.FromDays(7); /// - /// Gets or sets the base delay between retry attempts for outbox delivery and inbox processing. + /// Gets or sets the delay before the first retry of a failed outbox delivery or inbox processing attempt. /// /// /// - /// The outbox applies exponential backoff from this duration after each failed delivery attempt, including when - /// the target inbox returns DeliveryResult.Backpressured(). The inbox applies the same backoff after a + /// The outbox doubles this interval after each failed delivery attempt, including when the target inbox returns + /// DeliveryResult.Backpressured(), up to 64 times this interval. The inbox applies the same backoff after a /// handler fails. /// /// @@ -94,9 +94,9 @@ public class DurableMessagingOptions /// /// /// - /// The base backpressure retry delay. Must be greater than zero. Defaults to 1 second. + /// The initial retry interval. Must be greater than zero. Defaults to 1 second. /// - public TimeSpan BackpressureRetryDelay { get; set; } = TimeSpan.FromSeconds(1); + public TimeSpan RetryInitialInterval { get; set; } = TimeSpan.FromSeconds(1); /// /// Gets or sets the maximum number of attempts before an inbox message is dead-lettered. @@ -109,17 +109,17 @@ public class DurableMessagingOptions /// /// /// - /// The period starts with the first failed attempt and restarts whenever the receiver answers with + /// The timeout starts with the first failed attempt and restarts whenever the receiver answers with /// DeliveryResult.Backpressured(), so the outbox retries a backpressured message until the receiver takes it. - /// A receiver checks its deduplication records before answering, so only attempts since the period started can have - /// been accepted, and the period stays within to keep their retries deduplicated. + /// A receiver checks its deduplication records before answering, so only attempts since the timeout started can have + /// been accepted, and the timeout stays within to keep their retries deduplicated. /// /// /// The outbox logs each DeliveryResult.HandlerNotFound() answer as an error. /// /// /// Must be greater than zero and less than . Defaults to 6 hours. - public TimeSpan MaxOutboxRetryAge { get; set; } = TimeSpan.FromHours(6); + public TimeSpan OutboxRetryTimeout { get; set; } = TimeSpan.FromHours(6); /// /// Gets or sets how long inbox and outbox dead letters are retained. @@ -148,7 +148,7 @@ public class DurableMessagingOptions /// /// The outbox calls each receiver through this pipeline, so its strategies, such as timeouts, retries, and circuit /// breakers, act within one delivery attempt. When an attempt fails, the outbox schedules the next attempt with - /// backoff and continues delivering other messages. Receiving inboxes + /// backoff and continues delivering other messages. Receiving inboxes /// deduplicate by message ID, so retrying a delivery whose outcome is unknown is safe. /// /// @@ -185,16 +185,16 @@ public void Validate() throw new ArgumentOutOfRangeException(nameof(DeduplicationWindow), DeduplicationWindow, "DeduplicationWindow must be greater than TimeSpan.Zero."); } - if (BackpressureRetryDelay <= TimeSpan.Zero) + if (RetryInitialInterval <= TimeSpan.Zero) { - throw new ArgumentOutOfRangeException(nameof(BackpressureRetryDelay), BackpressureRetryDelay, "BackpressureRetryDelay must be greater than TimeSpan.Zero."); + throw new ArgumentOutOfRangeException(nameof(RetryInitialInterval), RetryInitialInterval, "RetryInitialInterval must be greater than TimeSpan.Zero."); } - if (BackpressureRetryDelay > MaxSupportedRetryDelay) + if (RetryInitialInterval > MaxSupportedRetryDelay) { throw new ArgumentOutOfRangeException( - nameof(BackpressureRetryDelay), - BackpressureRetryDelay, - $"BackpressureRetryDelay must be less than or equal to {MaxSupportedRetryDelay}."); + nameof(RetryInitialInterval), + RetryInitialInterval, + $"RetryInitialInterval must be less than or equal to {MaxSupportedRetryDelay}."); } if (MaxProcessingAttempts <= 0) @@ -202,17 +202,17 @@ public void Validate() throw new ArgumentOutOfRangeException(nameof(MaxProcessingAttempts), MaxProcessingAttempts, "MaxProcessingAttempts must be greater than zero."); } - if (MaxOutboxRetryAge <= TimeSpan.Zero) + if (OutboxRetryTimeout <= TimeSpan.Zero) { - throw new ArgumentOutOfRangeException(nameof(MaxOutboxRetryAge), MaxOutboxRetryAge, "MaxOutboxRetryAge must be greater than TimeSpan.Zero."); + throw new ArgumentOutOfRangeException(nameof(OutboxRetryTimeout), OutboxRetryTimeout, "OutboxRetryTimeout must be greater than TimeSpan.Zero."); } - if (MaxOutboxRetryAge >= DeduplicationWindow) + if (OutboxRetryTimeout >= DeduplicationWindow) { throw new ArgumentOutOfRangeException( - nameof(MaxOutboxRetryAge), - MaxOutboxRetryAge, - "MaxOutboxRetryAge must be less than DeduplicationWindow."); + nameof(OutboxRetryTimeout), + OutboxRetryTimeout, + "OutboxRetryTimeout must be less than DeduplicationWindow."); } if (DeliveryResiliencePipeline is null) diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index c4d1970987e..05b338de1bb 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -8,7 +8,7 @@ namespace Orleans.Messaging; /// stages a reference entry which carries the message ID and delivery /// state, but no payload or receiver, so the payload is journaled once, in the application's state. The outbox keeps /// every delivery semantic of an ordinary message: delivery after the sending write is durable, retries with backoff, -/// dead-lettering after MaxOutboxRetryAge, wake-ups, and metrics. +/// dead-lettering after OutboxRetryTimeout, wake-ups, and metrics. /// /// /// The outbox calls when the message is sent and before every delivery attempt. The first diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 72039e2f5bc..762a20a8b2f 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -312,8 +312,6 @@ namespace Orleans.Messaging.Configuration { public partial class DurableMessagingOptions { - public System.TimeSpan BackpressureRetryDelay { get { throw null; } set { } } - public System.TimeSpan DeadLetterRetentionPeriod { get { throw null; } set { } } public System.TimeSpan DeduplicationWindow { get { throw null; } set { } } @@ -324,14 +322,16 @@ public partial class DurableMessagingOptions public int MaxCapacity { get { throw null; } set { } } - public System.TimeSpan MaxOutboxRetryAge { get { throw null; } set { } } - public int MaxProcessingAttempts { get { throw null; } set { } } public int MaxRetainedDeadLetters { get { throw null; } set { } } public int OutboxBatchSize { get { throw null; } set { } } + public System.TimeSpan OutboxRetryTimeout { get { throw null; } set { } } + + public System.TimeSpan RetryInitialInterval { get { throw null; } set { } } + public void Validate() { } } } diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index 702cc5961fd..e8b2057fe53 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -49,7 +49,7 @@ public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() Assert.Equal(1000, options.MaxCapacity); Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); - Assert.Equal(TimeSpan.FromHours(6), options.MaxOutboxRetryAge); + Assert.Equal(TimeSpan.FromHours(6), options.OutboxRetryTimeout); Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); Assert.Equal(5, options.MaxProcessingAttempts); Assert.Same(ResiliencePipeline.Empty, options.DeliveryResiliencePipeline); @@ -65,10 +65,10 @@ public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContrac { (nameof(DurableMessagingOptions.MaxCapacity), options => options.MaxCapacity = 0), (nameof(DurableMessagingOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), - (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.Zero), - (nameof(DurableMessagingOptions.BackpressureRetryDelay), options => options.BackpressureRetryDelay = TimeSpan.MaxValue), + (nameof(DurableMessagingOptions.RetryInitialInterval), options => options.RetryInitialInterval = TimeSpan.Zero), + (nameof(DurableMessagingOptions.RetryInitialInterval), options => options.RetryInitialInterval = TimeSpan.MaxValue), (nameof(DurableMessagingOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), - (nameof(DurableMessagingOptions.MaxOutboxRetryAge), options => options.MaxOutboxRetryAge = TimeSpan.Zero), + (nameof(DurableMessagingOptions.OutboxRetryTimeout), options => options.OutboxRetryTimeout = TimeSpan.Zero), (nameof(DurableMessagingOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), (nameof(DurableMessagingOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), (nameof(DurableMessagingOptions.InboxBatchSize), options => options.InboxBatchSize = 0), @@ -95,18 +95,18 @@ public void Validate_NullDeliveryResiliencePipeline_Fails() } [Fact] - public void Validate_MaxOutboxRetryAgeNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() + public void Validate_OutboxRetryTimeoutNotLessThanDeduplicationWindow_FailsAtBoundaryAndAbove() { - foreach (var retryAge in new[] { TimeSpan.FromHours(2), TimeSpan.FromHours(3) }) + foreach (var retryTimeout in new[] { TimeSpan.FromHours(2), TimeSpan.FromHours(3) }) { var options = new DurableMessagingOptions { DeduplicationWindow = TimeSpan.FromHours(2), - MaxOutboxRetryAge = retryAge, + OutboxRetryTimeout = retryTimeout, }; var exception = Assert.Throws(options.Validate); - Assert.Equal(nameof(DurableMessagingOptions.MaxOutboxRetryAge), exception.ParamName); + Assert.Equal(nameof(DurableMessagingOptions.OutboxRetryTimeout), exception.ParamName); Assert.Contains("less than DeduplicationWindow", exception.Message, StringComparison.Ordinal); } } From ec0fe2a95b7f3ae9978c33191a83acca0a23a312 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:12:37 -0700 Subject: [PATCH 23/95] fix(messaging): reject ill-formed UTF-16 in identities and header names Orleans string serialization replaces unpaired surrogates with U+FFFD, so distinct hierarchical keys, message IDs, or header names could collapse on the wire. HierarchicalKey construction and parsing now accept only well-formed UTF-16, MessageId inherits the guarantee, and envelope validation rejects header names with unpaired surrogates. --- src/Orleans.Messaging/Envelope.cs | 2 +- src/Orleans.Messaging/EnvelopeValidation.cs | 2 + src/Orleans.Messaging/HierarchicalKey.cs | 46 +++++++++++++--- src/Orleans.Messaging/MessageId.cs | 2 +- .../Contracts/EnvelopeContractTests.cs | 35 +++++++++++- .../Contracts/HierarchicalKeyTests.cs | 54 ++++++++++++++++++- .../Contracts/MessageIdTests.cs | 32 +++++++++++ 7 files changed, 163 insertions(+), 10 deletions(-) diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs index b14436531cf..8393beee452 100644 --- a/src/Orleans.Messaging/Envelope.cs +++ b/src/Orleans.Messaging/Envelope.cs @@ -27,7 +27,7 @@ public readonly struct Envelope public string? Subject { get; init; } /// Gets optional application-defined named byte-array headers. - /// Use an ordinal dictionary for case-sensitive header names. Keys are nonempty and arrays are non-null. + /// Use an ordinal dictionary for case-sensitive header names. Keys are nonempty, well-formed UTF-16 strings, and arrays are non-null. [Id(3)] public Dictionary? Headers { get; init; } diff --git a/src/Orleans.Messaging/EnvelopeValidation.cs b/src/Orleans.Messaging/EnvelopeValidation.cs index 05cc40b29d5..906929017cb 100644 --- a/src/Orleans.Messaging/EnvelopeValidation.cs +++ b/src/Orleans.Messaging/EnvelopeValidation.cs @@ -20,6 +20,8 @@ public static void Validate(Envelope envelope) foreach (var header in headers) { if (string.IsNullOrEmpty(header.Key)) throw new ArgumentException("Header keys must not be empty.", nameof(envelope)); + if (!HierarchicalKey.IsWellFormedUtf16(header.Key)) + throw new ArgumentException("Header keys must not contain unpaired surrogate characters.", nameof(envelope)); if (header.Value is null) throw new ArgumentException("Header values must not be null.", nameof(envelope)); } } diff --git a/src/Orleans.Messaging/HierarchicalKey.cs b/src/Orleans.Messaging/HierarchicalKey.cs index a1735eda2ab..bad1b77ea8c 100644 --- a/src/Orleans.Messaging/HierarchicalKey.cs +++ b/src/Orleans.Messaging/HierarchicalKey.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -12,6 +13,7 @@ namespace Orleans.Messaging; /// /// Create constructs literal segments and escapes slash and backslash characters exactly once. /// Parse reads the canonical escaped path. The key wraps its canonical string; equality and hashing use that string. Applications preserve the identity across retries. +/// Every key is well-formed UTF-16: surrogate characters appear only as high-low pairs, so distinct keys keep distinct serialized forms. /// [Immutable, Alias("Orleans.Messaging.HierarchicalKey")] [JsonConverter(typeof(HierarchicalKeyJsonConverter))] @@ -23,6 +25,9 @@ namespace Orleans.Messaging; /// The separator between canonical segments. public const char SegmentSeparator = '/'; + private const char HighSurrogateStart = '\uD800'; + private const char LowSurrogateEnd = '\uDFFF'; + private readonly string? _value; private HierarchicalKey(string canonical) => _value = canonical; @@ -71,10 +76,10 @@ public override void WriteAsPropertyName(Utf8JsonWriter writer, HierarchicalKey /// The nonempty literal segment. /// The segment identity. /// is null. - /// is empty. + /// is empty or contains an unpaired surrogate. public static HierarchicalKey Create(string value) { - ArgumentException.ThrowIfNullOrEmpty(value); + ValidateSegment(value); return new(Escape(value)); } @@ -82,7 +87,7 @@ public static HierarchicalKey Create(string value) /// The nonempty literal segments. /// A flat canonical identity which owns its immutable backing string. /// A segment is null. - /// The input or a segment is empty. + /// The input or a segment is empty, or a segment contains an unpaired surrogate. public static HierarchicalKey Create(params ReadOnlySpan values) { if (values.IsEmpty) @@ -92,7 +97,7 @@ public static HierarchicalKey Create(params ReadOnlySpan values) var builder = new StringBuilder(); foreach (var value in values) { - ArgumentException.ThrowIfNullOrEmpty(value); + ValidateSegment(value); if (builder.Length > 0) builder.Append(SegmentSeparator); AppendEscaped(builder, value); } @@ -104,11 +109,11 @@ public static HierarchicalKey Create(params ReadOnlySpan values) /// The child identity. /// This key is unset. /// is null. - /// is empty. + /// is empty or contains an unpaired surrogate. public HierarchicalKey CreateChildKey(string value) { EnsureSet(); - ArgumentException.ThrowIfNullOrEmpty(value); + ValidateSegment(value); return new(string.Concat(_value, "/", Escape(value))); } @@ -234,6 +239,31 @@ private void EnsureSet() if (IsDefault) throw new InvalidOperationException("The key must not be unset."); } + private static void ValidateSegment(string value, [CallerArgumentExpression(nameof(value))] string? paramName = null) + { + ArgumentException.ThrowIfNullOrEmpty(value, paramName); + if (!IsWellFormedUtf16(value)) throw new ArgumentException("The value must not contain unpaired surrogate characters.", paramName); + } + + // Orleans string serialization replaces unpaired surrogates, so accepting them would let distinct keys collide on the wire. + internal static bool IsWellFormedUtf16(ReadOnlySpan value) + { + var i = value.IndexOfAnyInRange(HighSurrogateStart, LowSurrogateEnd); + if (i < 0) return true; + for (; i < value.Length; i++) + { + if (char.IsHighSurrogate(value[i])) + { + if (++i == value.Length || !char.IsLowSurrogate(value[i])) return false; + } + else if (char.IsLowSurrogate(value[i])) + { + return false; + } + } + return true; + } + private static string Escape(string value) { if (value.AsSpan().IndexOfAny(EscapeCharacter, SegmentSeparator) < 0) return value; @@ -277,6 +307,10 @@ private static bool IsCanonical(ReadOnlySpan path) segmentLength = 0; continue; } + else if (char.IsSurrogate(path[i])) + { + if (!char.IsHighSurrogate(path[i]) || ++i == path.Length || !char.IsLowSurrogate(path[i])) return false; + } segmentLength++; } return segmentLength != 0; diff --git a/src/Orleans.Messaging/MessageId.cs b/src/Orleans.Messaging/MessageId.cs index c72dd23b41c..f6082990280 100644 --- a/src/Orleans.Messaging/MessageId.cs +++ b/src/Orleans.Messaging/MessageId.cs @@ -30,7 +30,7 @@ namespace Orleans.Messaging; /// The nonempty literal segments. /// The message identity. /// A segment is null. - /// The input or a segment is empty. + /// The input or a segment is empty, or a segment contains an unpaired surrogate. public static MessageId Create(params ReadOnlySpan segments) => new(HierarchicalKey.Create(segments)); /// diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 1c6d2f2f309..29c81175fff 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -317,12 +317,23 @@ public void IdentityAndSubjectAdmissionBoundaries(string field, int length, bool [InlineData("invalid-subject")] [InlineData("empty-header-key")] [InlineData("null-header-value")] + [InlineData("lone-high-surrogate-header-key")] + [InlineData("lone-low-surrogate-header-key")] + [InlineData("reversed-surrogate-header-key")] public void InvalidMetadataIsRejectedBeforeAdmission(string invalid) { var envelope = Create([1]) with { Subject = invalid == "empty-subject" ? "" : invalid == "invalid-subject" ? "\ud800" : null, - Headers = invalid == "empty-header-key" ? new() { [""] = "value"u8.ToArray() } : invalid == "null-header-value" ? new() { ["key"] = null! } : null + Headers = invalid switch + { + "empty-header-key" => new() { [""] = "value"u8.ToArray() }, + "null-header-value" => new() { ["key"] = null! }, + "lone-high-surrogate-header-key" => new() { ["trace-\ud83d"] = "value"u8.ToArray() }, + "lone-low-surrogate-header-key" => new() { ["\ude00-trace"] = "value"u8.ToArray() }, + "reversed-surrogate-header-key" => new() { ["trace-\ude00\ud83d"] = "value"u8.ToArray() }, + _ => null + } }; Assert.ThrowsAny(() => Validate(envelope)); Assert.ThrowsAny(() => Validate(new InboxMessage { Envelope = envelope })); @@ -330,6 +341,28 @@ public void InvalidMetadataIsRejectedBeforeAdmission(string invalid) Assert.Equal(new byte[] { 1 }, envelope.Payload); } + [Fact] + public void SurrogatePairHeaderKeysRoundTripThroughOrleansSerialization() + { + using var sending = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var receiving = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var envelope = Create([1]) with + { + MessageId = MessageId.Create("order-\U0001F600"), + Headers = new(StringComparer.Ordinal) { ["trace-\U0001F600"] = "first"u8.ToArray(), ["trace-\U0001F601"] = "second"u8.ToArray() } + }; + Validate(envelope); + + var decoded = receiving.GetRequiredService>().Deserialize(sending.GetRequiredService>().SerializeToArray(envelope)); + + Validate(decoded); + Assert.Equal(envelope.MessageId, decoded.MessageId); + Assert.NotNull(decoded.Headers); + Assert.Equal(["trace-\U0001F600", "trace-\U0001F601"], decoded.Headers.Keys.Order(StringComparer.Ordinal)); + Assert.Equal("first"u8.ToArray(), decoded.Headers["trace-\U0001F600"]); + Assert.Equal("second"u8.ToArray(), decoded.Headers["trace-\U0001F601"]); + } + private static Envelope Create(byte[] payload) => new() { MessageId = Id, Payload = payload }; private static void Validate(T message) => Invoke("EnvelopeValidation", "Validate", message!); private static T Invoke(string type, string method, params object[] args) diff --git a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs index cb82497d4f4..c19c7905421 100644 --- a/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/HierarchicalKeyTests.cs @@ -220,7 +220,7 @@ public void Create_WithValues_EmptyLiteralSegmentRejects(int position) } [Fact] - public void Create_LiteralSegment_RejectsOnlyNullAndEmpty() + public void Create_LiteralSegment_RejectsNullAndEmpty() { Assert.Throws(() => HierarchicalKey.Create((string)null!)); Assert.Throws(() => HierarchicalKey.Create("")); @@ -255,6 +255,56 @@ public void Parse_WithNullString_ThrowsArgumentNullException() Assert.True(key.IsDefault); } + // Theory data replaces unpaired surrogates during discovery serialization, so cases use placeholders. + [Theory] + [InlineData("order-{high}")] + [InlineData("{low}-order")] + [InlineData("order-{low}{high}")] + [InlineData("{high}\U0001F600")] + public void Create_UnpairedSurrogate_ThrowsArgumentException(string template) + { + var literal = WithUnpairedSurrogates(template); + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create(literal)).ParamName); + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create("orders", literal)).ParamName); + Assert.Equal("value", Assert.Throws(() => HierarchicalKey.Create("orders").CreateChildKey(literal)).ParamName); + } + + [Theory] + [InlineData("order-{high}")] + [InlineData("{low}-order")] + [InlineData("order-{low}{high}")] + [InlineData("orders/{high}")] + [InlineData("orders/{low}")] + [InlineData("orders\\/{high}/x")] + public void Parse_UnpairedSurrogate_RejectsStringAndSpan(string template) + { + var path = WithUnpairedSurrogates(template); + Assert.Throws(() => HierarchicalKey.Parse(path)); + Assert.Throws(() => HierarchicalKey.Parse(path.AsSpan())); + Assert.False(HierarchicalKey.TryParse(path, null, out var first)); + Assert.True(first.IsDefault); + Assert.False(HierarchicalKey.TryParse(path.AsSpan(), null, out var second)); + Assert.True(second.IsDefault); + } + + [Fact] + public void SurrogatePairs_RoundTripThroughParseAndOrleansSerialization() + { + var key = HierarchicalKey.Create("order-\U0001F600", "\U0001F601/x"); + const string canonical = "order-\U0001F600/\U0001F601\\/x"; + Assert.Equal(canonical, key.ToString()); + Assert.Equal(2, key.CountSegments()); + Assert.Equal(key, HierarchicalKey.Parse(canonical)); + Assert.True(HierarchicalKey.TryParse(canonical.AsSpan(), null, out var parsed)); + Assert.Equal(key, parsed); + + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var decoded = second.GetRequiredService>().Deserialize(first.GetRequiredService>().SerializeToArray(key)); + Assert.Equal(canonical, decoded.ToString()); + Assert.NotEqual(decoded, HierarchicalKey.Create("order-\U0001F601", "\U0001F601/x")); + } + [Theory] [InlineData("a/b/c", 3)] [InlineData(@"a\/b/c", 2)] @@ -404,6 +454,8 @@ public void LongHierarchy_ConstructionNavigationAndFormatting_AreIterative() [Fact] public void SegmentSeparator_IsForwardSlash() => Assert.Equal('/', HierarchicalKey.SegmentSeparator); + internal static string WithUnpairedSurrogates(string template) => template.Replace("{high}", "\ud83d").Replace("{low}", "\ude00"); + private static List Segments(HierarchicalKey key) { var values = new List(); diff --git a/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs index fa53e6dc82e..ef6128f46c5 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessageIdTests.cs @@ -107,6 +107,38 @@ public void Json_DefaultValueRoundTripsAsNullAndIsRejectedAsDictionaryKey() public void Json_MalformedValueIsRejected(string json) => Assert.Throws(() => JsonSerializer.Deserialize(json)); + [Theory] + [InlineData("order-{high}")] + [InlineData("{low}-order")] + [InlineData("order-{low}{high}")] + public void UnpairedSurrogate_IsRejectedOnEveryConstructionPath(string template) + { + var text = HierarchicalKeyTests.WithUnpairedSurrogates(template); + Assert.Throws(() => MessageId.Create(text)); + Assert.Throws(() => MessageId.Create("orders", text)); + Assert.Throws(() => MessageId.Parse(text)); + Assert.Throws(() => MessageId.Parse(text.AsSpan())); + Assert.False(MessageId.TryParse(text, null, out var result)); + Assert.True(result.IsDefault); + Assert.False(MessageId.TryParse(text.AsSpan(), null, out result)); + Assert.True(result.IsDefault); + } + + [Fact] + public void SurrogatePairs_RoundTripThroughOrleansSerialization() + { + using var first = new ServiceCollection().AddSerializer().BuildServiceProvider(); + using var second = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var id = MessageId.Create("order-\U0001F600"); + + var decoded = second.GetRequiredService>().Deserialize(first.GetRequiredService>().SerializeToArray(id)); + + Assert.Equal(id, decoded); + Assert.Equal("order-\U0001F600", decoded.ToString()); + Assert.Equal(id, MessageId.Parse("order-\U0001F600")); + Assert.NotEqual(decoded, MessageId.Create("order-\U0001F601")); + } + [Theory] [InlineData(false)] [InlineData(true)] From e9b4f8c4ff8b778ff3247c4b72fbc41f58d1497c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:16:55 -0700 Subject: [PATCH 24/95] feat(messaging)!: make inbox handler selection read-only IInboxHandler.CanHandle now receives the InboxMessage instead of the mutable IInboxHandlerContext. A handler can only stage a terminal outcome from HandleAsync, after the inbox selects it. BREAKING CHANGE: IInboxHandler.CanHandle(IInboxHandlerContext) is replaced by CanHandle(InboxMessage). Orleans.Messaging is alpha and has no package validation baseline. --- src/Orleans.Messaging/IInboxHandler.cs | 7 ++++--- src/Orleans.Messaging/InboxDispatcher.cs | 2 +- src/Orleans.Messaging/InboxExtensions.cs | 4 ++-- src/Orleans.Messaging/README.md | 5 ++++- src/api/Orleans.Messaging/Orleans.Messaging.cs | 2 +- .../Contracts/HandlerRoutingContractTests.cs | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Orleans.Messaging/IInboxHandler.cs b/src/Orleans.Messaging/IInboxHandler.cs index 843d665ecfe..9d5d707a9ac 100644 --- a/src/Orleans.Messaging/IInboxHandler.cs +++ b/src/Orleans.Messaging/IInboxHandler.cs @@ -6,14 +6,15 @@ namespace Orleans.Messaging; /// Handles inbox messages which it accepts. /// /// The inbox dispatches each message to the first registered handler whose returns -/// . +/// . Selection receives only the message, so a handler stages a terminal outcome only +/// from , after the inbox selects it. /// public interface IInboxHandler { /// Determines whether this handler handles the message. - /// The message and its terminal operations. + /// The message to route. /// when this handler handles the message. - bool CanHandle(IInboxHandlerContext context); + bool CanHandle(InboxMessage message); /// Handles the message. /// The message and its terminal operations. diff --git a/src/Orleans.Messaging/InboxDispatcher.cs b/src/Orleans.Messaging/InboxDispatcher.cs index 3eea4acfb3a..476b63ed380 100644 --- a/src/Orleans.Messaging/InboxDispatcher.cs +++ b/src/Orleans.Messaging/InboxDispatcher.cs @@ -25,7 +25,7 @@ internal ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken c cancellationToken.ThrowIfCancellationRequested(); foreach (var handler in _handlers) { - if (handler.CanHandle(context)) + if (handler.CanHandle(context.Message)) { return handler.HandleAsync(context, cancellationToken); } diff --git a/src/Orleans.Messaging/InboxExtensions.cs b/src/Orleans.Messaging/InboxExtensions.cs index ac67ff879c8..a06ed484b88 100644 --- a/src/Orleans.Messaging/InboxExtensions.cs +++ b/src/Orleans.Messaging/InboxExtensions.cs @@ -76,8 +76,8 @@ public static IInbox Register( private abstract class Handler(MessageType messageType) : IInboxHandler { - public bool CanHandle(IInboxHandlerContext context) => - string.Equals(context.Message.Envelope.Subject, messageType.Subject, StringComparison.Ordinal); + public bool CanHandle(InboxMessage message) => + string.Equals(message.Envelope.Subject, messageType.Subject, StringComparison.Ordinal); public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index c79c4ee29bd..d2a04c4cd63 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -29,7 +29,10 @@ to navigate them. ## Handle messages `IInbox.Register(IInboxHandler)` adds a handler. Each message goes to the first -registered handler whose `CanHandle` returns `true`. +registered handler whose `CanHandle(InboxMessage)` returns `true`. Selection reads +only the message; the selected handler finishes it in `HandleAsync`. Handlers can be +registered at any time; registering during construction or activation makes them +available to recovered messages. `InboxExtensions.Register` binds a `MessageType` subject to a typed method: diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 762a20a8b2f..426e51b416b 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -155,7 +155,7 @@ public partial interface IInbox public partial interface IInboxHandler { - bool CanHandle(IInboxHandlerContext context); + bool CanHandle(InboxMessage message); System.Threading.Tasks.ValueTask HandleAsync(IInboxHandlerContext context, System.Threading.CancellationToken cancellationToken); } diff --git a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs index b87b63e618d..60391b581a7 100644 --- a/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/HandlerRoutingContractTests.cs @@ -100,7 +100,7 @@ private sealed class FakeContext(InboxMessage message) : IInboxHandlerContext private sealed class ProbeHandler(string name, bool canHandle, List order) : IInboxHandler { - public bool CanHandle(IInboxHandlerContext context) + public bool CanHandle(InboxMessage message) { order.Add($"{name}?"); return canHandle; From 74758fe5df483b3416123a345836e1a40ef80d27 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:17:08 -0700 Subject: [PATCH 25/95] docs(messaging): describe DeliverAt acceptance and capacity in the README The Envelope field list states that a scheduled message is accepted immediately, processed once the Durable Jobs clock reaches DeliverAt, and does not count toward MaxCapacity. --- src/Orleans.Messaging/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index d2a04c4cd63..0e06852d74a 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -15,6 +15,9 @@ typed message bindings, delivery results, and options. - `Subject`: an optional ordinal name, at most 256 UTF-8 bytes, that selects a typed handler. - `Headers`: optional named byte arrays with nonempty keys and non-null values. +- `DeliverAt`: an optional time on the Durable Jobs clock. The receiver accepts the + message immediately and processes it once that clock reaches `DeliverAt`. A message + with `DeliverAt` does not count toward `MaxCapacity`, including after it becomes due. `OutboxMessage` pairs an envelope with its receiver. `InboxMessage` carries the same envelope at the receiver. Keep published payload and header arrays unchanged. From 9d1a30ebc8b12b6a232818e3426ec3bcf6620a85 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:18:24 -0700 Subject: [PATCH 26/95] build(messaging): enable trim, AOT, and single-file analyzers Matches dotnet/orleans#11513: the net10.0 build of Orleans.Messaging runs the trim, NativeAOT, and single-file analyzers. The package builds without IL warnings. --- src/Orleans.Messaging/Orleans.Messaging.csproj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj index dd2b37b2bbf..801ea472629 100644 --- a/src/Orleans.Messaging/Orleans.Messaging.csproj +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -17,6 +17,12 @@ true + + true + true + true + + From 7fe9f4cff178ef145d759e0c8d11d0bd59b9bc3f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 14:44:59 -0700 Subject: [PATCH 27/95] feat(journaling): add journaled state hooks and write preparation Adds IJournaledStateHook registration on IJournaledStateManager, IStateMachine.PrepareWriteAsync, and pre/post-commit exceptions so features such as the durable inbox can establish persistence prerequisites before a write and observe its outcome. --- .../CompatibilitySuppressions.xml | 28 + src/Orleans.Journaling/IJournaledStateHook.cs | 63 + .../IJournaledStateManager.cs | 48 +- src/Orleans.Journaling/IStateMachine.cs | 19 + .../JournaledStateManager.Hooks.cs | 105 ++ .../JournaledStateManager.cs | 386 ++-- .../JournaledStatePostCommitException.cs | 22 + .../JournaledStatePreCommitException.cs | 22 + src/Orleans.Journaling/README.md | 64 + .../Orleans.Journaling/Orleans.Journaling.cs | 80 + .../DurableListJournalBenchmarks.cs | 17 + .../JournaledJobShardDiscoveryTests.cs | 4 + .../CodecRecoveryTests.cs | 24 +- .../DurableCollectionDirectWriteTests.cs | 17 + .../DurableListDirectWriteTests.cs | 17 + .../DurableStateManagerTests.cs | 2 +- .../KeyedJournalingRegistrationTests.cs | 4 + .../StateManagerHookTests.cs | 1566 +++++++++++++++++ .../StateManagerPreparationTests.cs | 282 +++ 19 files changed, 2657 insertions(+), 113 deletions(-) create mode 100644 src/Orleans.Journaling/IJournaledStateHook.cs create mode 100644 src/Orleans.Journaling/JournaledStateManager.Hooks.cs create mode 100644 src/Orleans.Journaling/JournaledStatePostCommitException.cs create mode 100644 src/Orleans.Journaling/JournaledStatePreCommitException.cs create mode 100644 test/Orleans.Journaling.Tests/StateManagerHookTests.cs create mode 100644 test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index eb687e7cf91..5e86005fc7b 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -239,6 +239,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -246,6 +253,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) @@ -274,6 +288,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -281,6 +302,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs new file mode 100644 index 00000000000..d8395a31615 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -0,0 +1,63 @@ +namespace Orleans.Journaling; + +/// +/// Identifies a journal persistence operation. +/// +public enum JournaledStateOperation +{ + /// Appends pending journal entries. + Write, + + /// Replaces storage with a snapshot. + Snapshot, + + /// Deletes storage and resets registered states. + Delete +} + +/// +/// Participates in the prerequisites and completion of actual journal operations. +/// +/// +/// Before hooks run in reverse list order on the owner's logical execution context, outside its lock. +/// The hook at index zero runs last among before hooks. Write operations then prepare the registered +/// states before capture; deletion proceeds directly from the before hooks. +/// Prerequisites remain valid until capture or deletion, covering changes staged during asynchronous +/// preparation and while the journal work loop resumes. After hooks run in forward list order after +/// storage acknowledgement and state acknowledgement or reset, including successful writes which +/// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later +/// pending changes separate. Hook implementations must complete without calling initialization, +/// persistence, or disposal on their own journal owner. Those operations are serialized behind the +/// current operation, so awaiting them from a hook would create a circular dependency. +/// Dependencies between journal owners must also remain acyclic. +/// Callbacks can request a hook registration change, observing its queued completion outside the +/// callback rather than awaiting work which follows the current operation. +/// +public interface IJournaledStateHook +{ + /// + /// Establishes prerequisites before capture or storage deletion. + /// + /// The operation about to execute. + /// The token for the owned operation's lifetime. + /// A completion representing the prerequisite work. + /// + /// Failure reports and retains pending changes + /// for an explicit retry. All staged changes remain safe to commit. Full deletion requires the + /// owner to stop admission and drain feature operations before queuing deletion. + /// + ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; + + /// + /// Completes post-persistence work after the operation succeeds. + /// + /// The completed operation. + /// The token for the owned operation's lifetime. + /// A completion representing post-persistence work. + /// + /// Every after hook is invoked even when an earlier after hook fails. Failures are surfaced as + /// and leave the successfully persisted owner usable. + /// Durable feature state supplies recovery for interrupted post-persistence work. + /// + ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; +} diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index b6363a1b4cc..9bce3b22a4b 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -8,10 +8,44 @@ namespace Orleans.Journaling; /// /// The owner registers state machines and initializes the journal before using recovered state. /// State machine instances and their dependencies retain the lifetime assigned by their caller. -/// Disposing this manager stops journal processing and releases its journal resources. +/// Disposal drains owned storage and hook operations before releasing resources. /// public interface IJournaledStateManager : IAsyncDisposable { + /// + /// Adds a hook before the next persistence operation which follows this registration. + /// + /// The hook to register. + /// The insertion index, or to append. + /// The token used to cancel the caller's wait. + /// A completion representing application of the registration. + /// + /// Registration completes synchronously before initialization begins, supporting constructor registration. + /// After initialization begins, idle registration applies synchronously; otherwise it is queued + /// between owned operations, after the current operation's after hooks finish. + /// Before hooks execute in reverse registration order; after hooks execute in forward order. + /// Register the final before hook at index zero. Writes then prepare the registered states before capture. + /// Registration is independent of state-machine registration and persists through recovery and deletion. + /// Every implementation provides registration and invokes hooks at the documented operation boundaries. + /// Queued registration is a write-coalescing barrier. + /// Cancellation ends the caller's wait while admitted registration remains owned by the manager. + /// Synchronous state callbacks can request registration and observe its completion outside the callback. + /// Await registration before staging changes whose safety depends on the added hook. + /// + ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default); + + /// + /// Removes a hook between persistence operations. + /// + /// The registered hook to remove. + /// The token used to cancel the caller's wait. + /// A completion representing application of the removal. + /// The hook is not registered when the removal is applied. + /// + /// Removal has the same ownership and ordering guarantees as . + /// + ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default); + /// ValueTask IAsyncDisposable.DisposeAsync() => default; @@ -51,8 +85,13 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Stage mutations only after establishing that they are safe to commit. Pending changes are shared /// by all callers using this manager. Storage acknowledgement establishes durability. - /// A failed journal operation fences the manager; recovery requires a new manager and state machine instances. + /// The manager invokes each state's after before hooks, + /// completing a synchronous ready pass before capture. + /// A storage or state-processing failure fences the manager; recovery requires a new manager and state machine instances. /// Cancellation stops the caller's wait; an already queued write continues to its storage outcome. + /// State preparation or before-hook failure reports and retains pending + /// changes for an explicit retry. After-hook failure reports + /// after successful persistence. /// /// The token used to cancel the caller's wait. /// A task representing the write acknowledgement. @@ -64,7 +103,10 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. - /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. + /// A storage or state-reset failure permanently fences the manager and requests deactivation of its owning grain. + /// Before-hook failure reports with state retained. + /// After-hook failure reports after storage deletion + /// and state reset succeed. /// /// The cancellation token. /// A which represents the operation. diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index bf3fd61204e..f0c07c397ff 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -15,6 +15,7 @@ namespace Orleans.Journaling; /// /// /// When the application requests a write, the journaled state manager calls +/// to establish the state's persistence prerequisites, then /// (and occasionally ) to materialize /// the pending changes, then flushes the journal to durable storage. /// @@ -68,6 +69,24 @@ public interface IStateMachine /// void OnRecoveryCompleted() { } + /// + /// Establishes the persistence prerequisites for this state's pending changes. + /// + /// The token which cancels preparation when the journal owner stops. + /// A completion representing readiness to capture this state. + /// + /// The manager invokes preparation after before hooks and before capturing an append or snapshot. + /// Preparation is idempotent and prepares this state's own prerequisites. After asynchronous + /// preparation completes, the manager repeats preparation for all states to include changes + /// staged on interleaved turns. Prepared prerequisites complete synchronously on subsequent calls + /// and remain valid through capture. A complete synchronous pass flows directly into capture. + /// Preparation completes without awaiting another operation on its own journal manager, + /// since that operation is serialized behind the current write. + /// A failure reports and retains pending changes + /// for an explicit write retry. + /// + ValueTask PrepareWriteAsync(CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + /// /// Writes pending state changes to the journal. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs new file mode 100644 index 00000000000..5b5c68fd943 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -0,0 +1,105 @@ +using System.Runtime.ExceptionServices; + +namespace Orleans.Journaling; + +internal partial class JournaledStateManager +{ + private List? _hooks; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + MutateHookAsync(hook, index, remove: false, cancellationToken); + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + MutateHookAsync(hook, index: null, remove: true, cancellationToken); + + private ValueTask MutateHookAsync(IJournaledStateHook hook, int? index, bool remove, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(hook); + cancellationToken.ThrowIfCancellationRequested(); + Task task; + lock (_lock) + { + ThrowIfFenced(); + ObjectDisposedException.ThrowIf(_disposed != 0, this); + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + if (_workLoop is null || (!_workItemRunning && _workQueue.Count == 0)) + { + ApplyHookMutation(hook, index, remove); + return default; + } + + var workItem = new HookMutationWorkItem(hook, index, remove); + _workQueue.Enqueue(workItem); + task = workItem.Task; + } + + _workSignal.Signal(); + return new(task.WaitAsync(cancellationToken)); + } + + private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove) + { + var hooks = _hooks ??= new(); + if (remove) + { + if (!hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + } + else + { + hooks.Insert(index ?? hooks.Count, hook); + } + } + + private ValueTask InvokeBeforeHooksAsync(ref int index, JournaledStateOperation operation, CancellationToken cancellationToken) + { + while (index >= 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var pending = _hooks![index--].BeforeOperationAsync(operation, cancellationToken); + if (!pending.IsCompleted) + { + return pending; + } + + pending.GetAwaiter().GetResult(); + } + + return default; + } + + private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + List? failures = null; + for (var i = 0; i < _hooks!.Count; i++) + { + try + { + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); + } + } + + if (failures is { Count: 1 }) + { + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + } + + if (failures is not null) + { + throw new AggregateException(failures); + } + } + + private sealed class HookMutationWorkItem(IJournaledStateHook hook, int? index, bool remove) : WorkItem + { + public IJournaledStateHook Hook { get; } = hook; + public int? Index { get; } = index; + public bool Remove { get; } = remove; + } +} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index cbf6be752bc..931f9bc6d21 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using Orleans.Diagnostics; using Orleans.Serialization.Buffers; @@ -28,11 +29,15 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private readonly StateDirectory _journalStreamDirectory; private readonly RetiredStateTracker _retirementTracker; private Task? _workLoop; + // Under _lock, covers recovery and dequeued work through the last after callback. + private bool _workItemRunning; private ManagerState _state; private long _deletionGeneration; private Exception? _failure; - private bool _migrationSnapshotRequired; + // A snapshot requirement not yet claimed by a write. Consume under the lock before preparation awaits. + private bool _snapshotRequired; private int _disposed; + private Task? _disposeTask; public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) : this(shared, CreateStorage(storageProvider, CreateJournalId(grainContext)), grainContext) @@ -98,6 +103,8 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal string WriteJournalFormatKey => _shared.JournalFormatKey; + private bool IsSnapshotRequired => _snapshotRequired || _storage.IsCompactionRequested; + internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine) @@ -127,7 +134,7 @@ protected void EnsureRegistrationAllowed() _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (_workLoop is not null) { - throw new InvalidOperationException("New states cannot be registered after journaled state manager initialization has begun."); + throw new InvalidOperationException("Journal registrations cannot be changed after journaled state manager initialization has begun."); } } } @@ -164,10 +171,11 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def ThrowIfFenced(); if (_workLoop is null) { + _workItemRunning = true; _workLoop = Start(); } - task = EnqueueOrGetPendingWorkItem(out didEnqueue); + task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -189,6 +197,12 @@ private async Task WorkLoop() await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.ForceYielding); while (!_shutdownCancellation.IsCancellationRequested) { + lock (_lock) + { + _workItemRunning = true; + } + + Exception? recoveryFailure = null; try { await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -201,15 +215,17 @@ private async Task WorkLoop() } catch (Exception exception) { - try - { - LogErrorProcessingWorkItems(_shared.Logger, exception); - } - finally + recoveryFailure = exception; + LogErrorProcessingWorkItems(_shared.Logger, exception); + } + finally + { + lock (_lock) { - lock (_lock) + _workItemRunning = false; + if (recoveryFailure is not null) { - FaultQueuedWorkItemsUnderLock(exception); + FaultQueuedWorkItemsUnderLock(recoveryFailure); } } } @@ -245,6 +261,11 @@ private async Task WorkLoop() WorkItem workItem; lock (_lock) { + if (_shutdownCancellation.IsCancellationRequested) + { + return; + } + if (!_workQueue.TryDequeue(out var dequeuedWorkItem)) { // Wait for the queue to be signaled again. @@ -252,59 +273,88 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; + _workItemRunning = true; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); - var queueOperation = GetStorageQueueOperation(workItem); - var queueDuration = queueOperation is null - ? default - : _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); + string? queueOperation = null; + TimeSpan queueDuration = default; var recordQueueDuration = workItem is DeleteStateWorkItem; - Activity? storageActivity = queueOperation is null - ? null - : ActivitySources.StorageGrainSource.StartActivity( - $"journal {queueOperation}", - ActivityKind.Internal, - workItem.TraceParent, - tags: null, - links: workItem.TraceLinks); - if (storageActivity is not null) - { - storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); - } + Activity? storageActivity = null; + JournaledStateOperation? operationType = null; + string? compactionReason = null; + var beforeCommitRunning = false; + var afterHookRunning = false; try { - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem - && workItem.DeletionGeneration != Volatile.Read(ref _deletionGeneration)) + lock (_lock) + { + if (workItem is WriteStateWorkItem + && workItem.DeletionGeneration != _deletionGeneration) + { + throw new InvalidOperationException( + "The journaled state operation was queued before deletion reset its state."); + } + + operationType = GetOperationType(workItem); + if (operationType == JournaledStateOperation.Snapshot) + { + compactionReason = _snapshotRequired + ? JournalingInstruments.CompactionReasonUserSnapshot + : JournalingInstruments.CompactionReasonStorageRequested; + _snapshotRequired = false; + } + } + + (queueOperation, queueDuration, storageActivity) = GetStorageQueueOperation(workItem, operationType, processingTimestamp); + + if (operationType is { } operation && _hooks is { Count: > 0 }) + { + beforeCommitRunning = true; + // Keep synchronous final completion and capture in the same work-loop continuation. + for (var i = _hooks.Count - 1; i >= 0;) + { + await InvokeBeforeHooksAsync(ref i, operation, _shutdownCancellation.Token).ConfigureAwait(true); + } + + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + beforeCommitRunning = false; + } + + if (workItem is WriteStateWorkItem) { - throw new InvalidOperationException( - "The journaled state operation was queued before deletion reset its state."); + beforeCommitRunning = true; + for (var prepared = false; !prepared;) + { + await PrepareStates(ref prepared, _shutdownCancellation.Token).ConfigureAwait(true); + } + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + beforeCommitRunning = false; } // Note that the implementation of each command is inlined to avoid allocating unnecessary async states. // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { - case AppendJournalWorkItem: - case WriteSnapshotWorkItem: + case HookMutationWorkItem mutation: + { + // The work loop owns the hook set while _workItemRunning is true. + ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); + break; + } + + case WriteStateWorkItem: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. - var isSnapshot = workItem is WriteSnapshotWorkItem - || _migrationSnapshotRequired - || _storage.IsCompactionRequested; + var isSnapshot = operationType == JournaledStateOperation.Snapshot; var operationLabel = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; if (isSnapshot) { - var compactionReason = workItem is WriteSnapshotWorkItem - ? JournalingInstruments.CompactionReasonUserSnapshot - : _migrationSnapshotRequired - ? JournalingInstruments.CompactionReasonMigration - : JournalingInstruments.CompactionReasonStorageRequested; - _shared.Instruments.OnCompactionTriggered(compactionReason); + _shared.Instruments.OnCompactionTriggered(compactionReason!); } _shared.Instruments.OnWriteCoalesced(operationLabel, workItem.CallerCount); var gatherStartTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -325,11 +375,6 @@ private async Task WorkLoop() RetireOrResurectStates(); } - if (_migrationSnapshotRequired) - { - ThrowIfMigrationBlockedByRetiredStates(); - } - // The map of state ids is itself stored as a durable state with the id 0. // This must be stored first, since it includes the identities of all other states, which are needed when replaying the journal. // If we removed retired states, this snapshot will persist that change. @@ -470,11 +515,6 @@ private async Task WorkLoop() { state.OnWriteCompleted(); } - - if (isSnapshot) - { - _migrationSnapshotRequired = false; - } } } @@ -540,13 +580,64 @@ private async Task WorkLoop() } } + if (operationType is { } completedOperation && _hooks is { Count: > 0 }) + { + afterHookRunning = true; + await InvokeAfterHooksAsync(completedOperation, _shutdownCancellation.Token).ConfigureAwait(true); + afterHookRunning = false; + } + if (recordQueueDuration && queueOperation is not null) { _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: true); } storageActivity?.SetStatus(ActivityStatusCode.Ok); - workItem.SetResult(); + lock (_lock) + { + _workItemRunning = false; + workItem.SetResult(); + } + } + catch (Exception exception) when (beforeCommitRunning || afterHookRunning) + { + if (beforeCommitRunning && operationType == JournaledStateOperation.Snapshot) + { + lock (_lock) + { + _snapshotRequired = true; + } + } + + if (beforeCommitRunning && exception is OperationCanceledException && _shutdownCancellation.IsCancellationRequested) + { + workItem.TrySetCanceled(_shutdownCancellation.Token); + continue; + } + + LogOperationHookFailed(_shared.Logger, exception, operationType!.Value, afterHookRunning); + storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal operation callback failed."); + if (recordQueueDuration && queueOperation is not null) + { + _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); + } + + lock (_lock) + { + _workItemRunning = false; + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(operationType.Value, exception) + : new JournaledStatePreCommitException(operationType.Value, exception)); + } + } + catch (Exception exception) when (workItem is HookMutationWorkItem) + { + LogHookMutationFailed(_shared.Logger, exception); + lock (_lock) + { + _workItemRunning = false; + workItem.SetException(exception); + } } catch (Exception exception) { @@ -580,6 +671,11 @@ private async Task WorkLoop() } finally { + lock (_lock) + { + _workItemRunning = false; + } + storageActivity?.Dispose(); } } @@ -596,6 +692,33 @@ private async Task WorkLoop() } } + private JournaledStateOperation? GetOperationType(WorkItem workItem) => workItem switch + { + WriteStateWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, + WriteStateWorkItem => JournaledStateOperation.Write, + DeleteStateWorkItem => JournaledStateOperation.Delete, + _ => null + }; + + private ValueTask PrepareStates(ref bool prepared, CancellationToken cancellationToken) + { + prepared = true; + lock (_lock) + { + foreach (var state in _states.Values) + { + var preparation = state.PrepareWriteAsync(cancellationToken); + if (!preparation.IsCompletedSuccessfully) + { + prepared = false; + return preparation; + } + preparation.GetAwaiter().GetResult(); + } + } + return ValueTask.CompletedTask; + } + private void Fence(Exception exception) { lock (_lock) @@ -668,20 +791,6 @@ private void RetireOrResurectStates() } } - private void ThrowIfMigrationBlockedByRetiredStates() - { - foreach (var state in _statesMap.Values) - { - if (state is RetiredStateMachine { PreservedEntries.Count: > 0 } retiredState) - { - throw new InvalidOperationException( - $"Cannot migrate journal to format key '{_shared.JournalFormatKey}' because stream " + - $"{retiredState.StreamId.Value} belongs to a state which is not currently registered. Register the state so it can be decoded " + - "and snapshotted in the configured format, or keep the previous journal format configured until the state can be retired."); - } - } - } - private JournalStreamWriter CreateJournalStreamWriter(JournalStreamId streamId) => _journalWriter.CreateJournalStreamWriter(streamId); private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWriter, bool isSnapshot, uint id, IStateMachine state) @@ -705,7 +814,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = de lock (_lock) { ThrowIfStateOperationsUnavailable(); - task = EnqueueOrGetPendingWorkItem(out didEnqueue); + task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -780,7 +889,7 @@ private async Task RecoverAsync(CancellationToken cancellationToken) private void ResetForRecovery() { _journalWriter.Reset(); - _migrationSnapshotRequired = false; + _snapshotRequired = false; _statesMap.Clear(); _statesMap[StateDirectory.Id] = _journalStreamDirectory; _statesMap[RetiredStateTracker.Id] = _retirementTracker; @@ -851,7 +960,7 @@ private void ProcessRecoveryBuffer(JournalBufferReader buffer, IJournalMetadata? { if (!string.Equals(journalFormatKey, _shared.JournalFormatKey, StringComparison.Ordinal)) { - _migrationSnapshotRequired = true; + _snapshotRequired = true; } var journalFormat = GetJournalFormat(journalFormatKey); @@ -894,11 +1003,9 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); - var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; - operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; - pendingWrite = isSnapshot - ? EnqueueOrGetPendingWorkItem(out didEnqueue) - : EnqueueOrGetPendingWorkItem(out didEnqueue); + _snapshotRequired = IsSnapshotRequired; + operation = _snapshotRequired ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; + pendingWrite = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -999,26 +1106,39 @@ private async ValueTask DeleteStorageAsync(CancellationToken cancellationToken) } } - private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) + private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) where TWorkItem : WorkItem, new() { + TWorkItem? pending = null; foreach (var workItem in _workQueue) { - if (workItem.GetType() != typeof(TWorkItem)) + if (workItem is HookMutationWorkItem) + { + // Later callers must not coalesce into work which precedes a registration change. + pending = null; + continue; + } + + if (pending is not null || workItem is not TWorkItem candidate) { continue; } - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem + if (workItem is WriteStateWorkItem && workItem.DeletionGeneration != _deletionGeneration) { continue; } - workItem.RecordTraceContext(); - workItem.AddCaller(); + pending = candidate; + } + + if (pending is not null) + { + pending.RecordTraceContext(); + pending.AddCaller(); didEnqueue = false; - return workItem.Task; + return pending; } var newWorkItem = new TWorkItem(); @@ -1027,7 +1147,7 @@ private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) newWorkItem.RecordTraceContext(); _workQueue.Enqueue(newWorkItem); didEnqueue = true; - return newWorkItem.Task; + return newWorkItem; } private void BindState(string name, uint id) @@ -1062,9 +1182,22 @@ void ILifecycleParticipant.Participate(IGrainLifecycle observer private async Task StopAsync(CancellationToken cancellationToken) { + AggregateException? cancellationFailure = null; lock (_lock) { - _shutdownCancellation.Cancel(); + try + { + _shutdownCancellation.Cancel(); + } + catch (AggregateException exception) + { + cancellationFailure = exception; + } + } + + if (cancellationFailure is not null) + { + LogShutdownCancellationFailed(_shared.Logger, cancellationFailure); } _workSignal.Signal(); @@ -1079,6 +1212,11 @@ private async Task StopAsync(CancellationToken cancellationToken) { CancelQueuedWorkItems(_shutdownCancellation.Token); } + + if (cancellationFailure is not null) + { + ExceptionDispatchInfo.Capture(cancellationFailure).Throw(); + } } private void CancelQueuedWorkItems(CancellationToken cancellationToken) @@ -1097,21 +1235,41 @@ void IDisposable.Dispose() DisposeAsync().AsTask().GetAwaiter().GetResult(); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) + lock (_lock) { - return; + if (_disposeTask is null) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _disposeTask = completion.Task; + _ = DisposeCoreAsync(completion); + } + + return new(_disposeTask); } + } + private async Task DisposeCoreAsync(TaskCompletionSource completion) + { + _disposed = 1; try { - await StopAsync(CancellationToken.None).ConfigureAwait(false); + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + finally + { + _shutdownCancellation.Dispose(); + _journalWriter.Dispose(); + } + + completion.SetResult(); } - finally + catch (Exception exception) { - _shutdownCancellation.Dispose(); - _journalWriter.Dispose(); + completion.SetException(exception); } } @@ -1193,20 +1351,37 @@ public void RecordTraceContext() } } - private static string? GetStorageQueueOperation(WorkItem workItem) => - workItem switch + private (string? Operation, TimeSpan QueueDuration, Activity? Activity) GetStorageQueueOperation( + WorkItem workItem, + JournaledStateOperation? operationType, + long processingTimestamp) + { + var operation = operationType switch { - AppendJournalWorkItem => JournalingInstruments.OperationAppend, - WriteSnapshotWorkItem => JournalingInstruments.OperationSnapshot, - DeleteStateWorkItem => JournalingInstruments.OperationDelete, + JournaledStateOperation.Snapshot => JournalingInstruments.OperationSnapshot, + JournaledStateOperation.Write => JournalingInstruments.OperationAppend, + JournaledStateOperation.Delete => JournalingInstruments.OperationDelete, _ => null }; + if (operation is null) + { + return default; + } - private sealed class InitializeWorkItem : WorkItem; + var queueDuration = _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); + var activity = ActivitySources.StorageGrainSource.StartActivity( + $"journal {operation}", + ActivityKind.Internal, + workItem.TraceParent, + tags: null, + links: workItem.TraceLinks); + activity?.SetTag(ActivityTagKeys.JournalStorageOperation, operation); + return (operation, queueDuration, activity); + } - private sealed class AppendJournalWorkItem : WorkItem; + private sealed class InitializeWorkItem : WorkItem; - private sealed class WriteSnapshotWorkItem : WorkItem; + private sealed class WriteStateWorkItem : WorkItem; private sealed class DeleteStateWorkItem : WorkItem; @@ -1330,8 +1505,6 @@ private sealed class RetiredStateMachine(JournalStreamId streamId) : IStateMachi public JournalStreamId StreamId { get; } = streamId; - public IReadOnlyList PreservedEntries => _preservedEntries; - void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => _preservedEntries.Add(new PreservedJournalEntry(entry.FormatKey, entry.Reader)); @@ -1352,6 +1525,21 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal {Operation} callback failed. Persistence completed: {Committed}.")] + private static partial void LogOperationHookFailed(ILogger logger, Exception exception, JournaledStateOperation operation, bool committed); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal operation hook registration change failed.")] + private static partial void LogHookMutationFailed(ILogger logger, Exception exception); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal shutdown cancellation callback failed; owned operations are drained before resources are released.")] + private static partial void LogShutdownCancellationFailed(ILogger logger, Exception exception); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/Orleans.Journaling/JournaledStatePostCommitException.cs b/src/Orleans.Journaling/JournaledStatePostCommitException.cs new file mode 100644 index 00000000000..8274c8726d6 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePostCommitException.cs @@ -0,0 +1,22 @@ +namespace Orleans.Journaling; + +/// +/// Reports a hook failure after the journal operation and its state acknowledgement or reset succeeded. +/// +/// +/// The journal owner remains usable. The caller handles the failed post-persistence work using +/// the feature's durable recovery protocol, preserving the completed business operation. +/// +[GenerateSerializer] +public sealed class JournaledStatePostCommitException : Exception +{ + /// + /// Initializes a post-persistence hook failure. + /// + /// The successfully completed journal operation. + /// The failure or aggregate of failures from after hooks. + public JournaledStatePostCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' completed, but a post-persistence hook failed.", innerException) + { + } +} diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs new file mode 100644 index 00000000000..e4ddb71939e --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -0,0 +1,22 @@ +namespace Orleans.Journaling; + +/// +/// Reports a persistence prerequisite failure which prevented the journal storage operation. +/// +/// +/// Pending changes remain staged and safe to commit. The caller can restore the prerequisite +/// and explicitly retry persistence, or retire its owner and recover from durable state. +/// +[GenerateSerializer] +public sealed class JournaledStatePreCommitException : Exception +{ + /// + /// Initializes a persistence prerequisite failure. + /// + /// The journal operation prevented by the failed prerequisite. + /// The original prerequisite failure. + public JournaledStatePreCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' was prevented by a persistence prerequisite failure.", innerException) + { + } +} diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 2f5046a30be..d711ebee8bf 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -207,15 +207,29 @@ provider and requested state name. - `Reset`: reset in-memory state and bind a `JournalStreamWriter`. - `ReplayEntry`: reconstruct state from a recorded operation. +- `PrepareWriteAsync`: establish asynchronous persistence prerequisites for pending state. - `WritePendingEntries`: emit pending operations into the supplied writer. - `WriteSnapshot`: emit a reconstructible snapshot into the supplied writer. - `OnRecoveryCompleted`: finish reconstruction before application use. - `OnWriteCompleted`: publish effects that depend on storage acknowledgement. +Command-backed states can use `IDurableValueCommandCodec` to encode a +complete command before applying it locally. A snapshot is a command in the same +application-defined schema which reconstructs the whole state. Annotate command data +for Orleans serialization and supply System.Text.Json metadata when using the JSON codec. + Recovery resets state machines and replays durable entries. A failed initialization can be retried on the same manager and registered states. `JournalReplayContext.ResolveStateMachine` routes entries to the state machine for their stream. +Write preparation runs after before hooks and before append or snapshot capture. +Each state prepares its own prerequisites idempotently. An asynchronous completion +leaves prepared prerequisites synchronously ready; the manager then rechecks all states +to include mutations staged on interleaved turns. A complete synchronous pass flows +directly into capture. Preparation failures retain pending changes for an explicit +write retry and report `JournaledStatePreCommitException`. The owner's shutdown token +cancels preparation, and disposal drains its actual completion. + `IJournaledStateManager` is independent of the grain-facing `IDurableStateManager` and extends `IAsyncDisposable`. Its owner API provides `RegisterStateMachine`, `TryGetStateMachine`, `InitializeAsync`, `WriteStateAsync`, whole-journal `DeleteStateAsync`, and `PendingWriteByteCount`. @@ -264,6 +278,56 @@ the operation's token before use, or deliberately supply them through a registra lifecycle ownership. Creation through the explicit-journal factory keeps failure handling independent of the ambient grain context, including when the caller subsequently enrolls the manager in a lifecycle. +## Journal operation hooks + +Use `IJournaledStateManager.AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to register +`IJournaledStateHook` callbacks. Registration completes synchronously before initialization, +supporting constructors. After initialization begins, changes apply synchronously when idle, +or queue between complete journal operations while busy. +Each operation keeps its hook set through preparation, capture, storage ACK and after callbacks. +Queued changes are write-coalescing barriers: writes admitted afterward observe the changed set. +Caller cancellation ends only its wait for an admitted change; the manager retains ownership. +Await registration before staging changes whose safety depends on the new hook. +Every journal owner implementation provides registration and invokes callbacks at the operation +boundaries described below. Delegating owners forward registration methods to their inner owner. +Synchronous state callbacks can request a queued registration and observe completion outside the callback. + +For each actual write, snapshot, or deletion, before hooks run in reverse list order outside +the manager lock. Add the final before hook with `index: 0`. Writes then prepare the +registered states before capture; deletion follows the before hooks. Prerequisites remain valid through capture or +deletion, covering changes staged during asynchronous preparation and while the work loop resumes. +When state preparation is synchronously ready, capture follows in the same work-loop +continuation. Other owner turns can stage changes while an incomplete callback is awaited, +so its prerequisite covers those changes through capture or deletion. +Keep operation-local bookkeeping for the captured batch separate from later pending changes. + +After hooks run in list order after storage acknowledgement and state acknowledgement or reset. +They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual +operation. Features implement the before and after callbacks on their own hook, retaining their +identity and operation-local bookkeeping there. +Queued writes coalesce into one operation. A snapshot requirement from any caller upgrades that +pending work to a snapshot, even if the storage compaction signal clears before it runs. +The manager consumes the pending requirement when selecting a write and restores it when a before +hook or state preparation fails. Storage acknowledgement leaves newer snapshot requests untouched. + +| Outcome | Owner and caller behavior | +| --- | --- | +| Before hook or state preparation fails | `JournaledStatePreCommitException` retains pending state for an explicit retry after restoring the prerequisite. | +| Storage or state processing fails | The original failure fences the manager; create a fresh owner and recover the durable outcome. | +| After hook fails | `JournaledStatePostCommitException` reports successful persistence. Every remaining after hook runs, failures are aggregated, and the manager stays usable. The feature's durable recovery protocol resumes interrupted post-persistence work. | + +Hooks receive the owner's shutdown token. Caller cancellation ends the caller's wait while +the owned prerequisite, capture, storage, and completion phases continue. Disposal cancels the +owner token and drains owned work before releasing journal resources, including when cancellation +callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. +Shutdown closes work admission and cancels queued operations while an already running operation +drains to its actual storage and hook outcome. +Hook implementations complete without calling initialization, persistence, or disposal on their +own owner: awaiting an operation serialized behind the current callback creates a circular dependency. +Dependencies between journal owners also remain acyclic. +Before whole-journal deletion, the feature owner stops admission and drains its own operations; +deletion completion follows storage deletion and registered-state reset. + ## State identity and retirement Preserve state names across activations and deployments. A stream absent from the setup declarations diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 396a1692cad..516ebaf4d04 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,13 +219,21 @@ public partial interface IDurableValue T? Value { get; set; } } + public partial interface IJournaledStateHook + { + System.Threading.Tasks.ValueTask AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + } + public partial interface IJournaledStateManager : System.IAsyncDisposable { long PendingWriteByteCount { get; } + System.Threading.Tasks.ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken = default); void RegisterStateMachine(string name, IStateMachine stateMachine); + System.Threading.Tasks.ValueTask RemoveHookAsync(IJournaledStateHook hook, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); bool TryGetStateMachine(string name, out IStateMachine? stateMachine); System.Threading.Tasks.ValueTask WriteStateAsync(System.Threading.CancellationToken cancellationToken = default); @@ -307,6 +315,7 @@ public partial interface IStateMachine { void OnRecoveryCompleted(); void OnWriteCompleted(); + System.Threading.Tasks.ValueTask PrepareWriteAsync(System.Threading.CancellationToken cancellationToken = default); void ReplayEntry(JournalEntry entry, JournalReplayContext context); void Reset(JournalStreamWriter writer); void WritePendingEntries(JournalStreamWriter writer); @@ -424,6 +433,25 @@ public sealed partial class JournaledStateManagerOptions public System.TimeSpan RetirementGracePeriod { get { throw null; } set { } } } + public enum JournaledStateOperation + { + Write = 0, + Snapshot = 1, + Delete = 2 + } + + [GenerateSerializer] + public sealed partial class JournaledStatePostCommitException : System.Exception + { + public JournaledStatePostCommitException(JournaledStateOperation operation, System.Exception innerException) { } + } + + [GenerateSerializer] + public sealed partial class JournaledStatePreCommitException : System.Exception + { + public JournaledStatePreCommitException(JournaledStateOperation operation, System.Exception innerException) { } + } + public readonly ref partial struct JournalEntry { private readonly object _dummy; @@ -800,10 +828,62 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. where TBufferWriter : System.Buffers.IBufferWriter { } } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePostCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePostCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePostCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePostCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePostCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePreCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePreCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePreCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePreCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePreCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Copier_DurableTaskCompletionSourceState : global::Orleans.Serialization.Cloning.ShallowCopier> { } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePostCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePreCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + } } \ No newline at end of file diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index c94bd6045e2..5cb36fbffcf 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -126,6 +126,23 @@ private void ReplayEncodedJournalData() private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter buffer, JournalStreamId streamId) : IJournaledStateManager { + private readonly List _hooks = []; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(buffer.CreateJournalStreamWriter(streamId)); diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index 7a9862c5cab..39e4213c5c1 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,6 +727,10 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + inner.AddHookAsync(hook, index, cancellationToken); + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + inner.RemoveHookAsync(hook, cancellationToken); public int DisposeCalls { get; private set; } public long PendingWriteByteCount => inner.PendingWriteByteCount; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); diff --git a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs index 10ac2a9537c..4e975de5f74 100644 --- a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs +++ b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs @@ -293,27 +293,31 @@ public async Task Recovery_EmptyJournalWithStaleMetadata_WritesConfiguredFormat( Assert.Contains("""[8,["set","alpha",1]]""", Encoding.UTF8.GetString(storage.Segments.Single()), StringComparison.Ordinal); } - [Fact] - public async Task Migration_WithUnregisteredRetiredState_ThrowsClearError() + [Theory] + [InlineData(OrleansBinaryJournalFormat.JournalFormatKey, JsonLinesJournalFormat.JournalFormatKey)] + [InlineData(JsonLinesJournalFormat.JournalFormatKey, OrleansBinaryJournalFormat.JournalFormatKey)] + public async Task Migration_WithUnregisteredRetiredState_PreservesStorage(string sourceFormat, string targetFormat) { - var storage = new VolatileJournalStorage(OrleansBinaryJournalFormat.JournalFormatKey); - using var first = CreateFormatAwareTestSystem(storage, OrleansBinaryJournalFormat.JournalFormatKey); - var dict = CreateFormatAwareDictionary(first, OrleansBinaryJournalFormat.JournalFormatKey, "dict"); + var storage = new VolatileJournalStorage(sourceFormat); + using var first = CreateFormatAwareTestSystem(storage, sourceFormat); + var dict = CreateFormatAwareDictionary(first, sourceFormat, "dict"); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); await first.Manager.WriteStateAsync(CancellationToken.None); - storage.SetConfiguredJournalFormatKey(JsonLinesJournalFormat.JournalFormatKey); - using var recovered = CreateFormatAwareTestSystem(storage, JsonLinesJournalFormat.JournalFormatKey); - var other = CreateFormatAwareDictionary(recovered, JsonLinesJournalFormat.JournalFormatKey, "other"); + var originalBytes = Assert.Single(storage.Segments).ToArray(); + storage.SetConfiguredJournalFormatKey(targetFormat); + using var recovered = CreateFormatAwareTestSystem(storage, targetFormat); + var other = CreateFormatAwareDictionary(recovered, targetFormat, "other"); await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); other.Add("beta", 2); var exception = await Assert.ThrowsAsync( () => recovered.Manager.WriteStateAsync(CancellationToken.None).AsTask()); - Assert.Contains("Cannot migrate journal", exception.Message, StringComparison.Ordinal); - Assert.Contains("not currently registered", exception.Message, StringComparison.Ordinal); + Assert.Contains("cannot append preserved entry", exception.Message, StringComparison.Ordinal); + Assert.Equal(sourceFormat, storage.StoredJournalFormatKey); + Assert.Equal(originalBytes, Assert.Single(storage.Segments)); } [Theory] diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 26cf16aac46..ede9edb2f56 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -193,6 +193,23 @@ public void Collections_UseDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + private readonly List _hooks = []; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index e1cc3708672..245561f8c50 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -92,6 +92,23 @@ public void Add_UsesDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + private readonly List _hooks = []; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs index b145f0f7417..e8355089172 100644 --- a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs @@ -631,7 +631,7 @@ private static void AssertIncompatible( private static void AssertAdmissionRejected(Action action) { var error = Assert.Throws(action); - Assert.Equal("New states cannot be registered after journaled state manager initialization has begun.", error.Message); + Assert.Equal("Journal registrations cannot be changed after journaled state manager initialization has begun.", error.Message); } private static void AssertFenced(InvalidOperationException error, IOException cause) diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index e9822dd9557..dd7945ec7cb 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,6 +746,10 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + inner.AddHookAsync(hook, index, cancellationToken); + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + inner.RemoveHookAsync(hook, cancellationToken); public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs new file mode 100644 index 00000000000..8833f87cb85 --- /dev/null +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -0,0 +1,1566 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Reflection; +using System.Threading.Channels; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Diagnostics; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Journaling.Tests; + +public partial class StateManagerTests +{ + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorageOutcome(bool snapshot, bool storageFails) + { + using var parent = new Activity("journal queue test").SetIdFormat(ActivityIdFormat.W3C).Start(); + var stopped = new ConcurrentBag(); + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == ActivitySources.StorageActivitySourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = activity => + { + if (activity.ParentSpanId == parent.SpanId) + { + stopped.Add(activity); + } + } + }; + ActivitySource.AddActivityListener(listener); + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage + { + IsCompactionRequested = snapshot, + NextAppendException = storageFails && !snapshot ? failure : null, + NextReplaceException = storageFails && snapshot ? failure : null + }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + if (storageFails) + { + Assert.Same(failure, await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + } + else + { + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + await manager.DisposeAsync(); + var activity = Assert.Single(stopped); + var operation = snapshot ? "snapshot" : "append"; + Assert.Equal($"journal {operation}", activity.OperationName); + Assert.Equal(operation, activity.GetTagItem(ActivityTagKeys.JournalStorageOperation)); + Assert.Equal(storageFails ? ActivityStatusCode.Error : ActivityStatusCode.Ok, activity.Status); + Assert.Equal(storageFails ? failure.Message : null, activity.StatusDescription); + } + + [Fact] + public async Task Hooks_IdleRegistrationAppliesSynchronously() + { + await using var manager = CreateTestSystem().Manager; + _ = new DurableValue("state", manager, CreateValueCodec()); + var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.Null(field.GetValue(manager)); + + var before = 0; + var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; + Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + await Assert.ThrowsAsync(() => manager.AddHookAsync(null!, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, before); + Assert.True(manager.AddHookAsync(hook, index: 0, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + + await using var withoutHooks = CreateTestSystem().Manager; + await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); + await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Null(field.GetValue(withoutHooks)); + } + + [Fact] + public async Task Hooks_ShutdownRejectsRegistrationAndRemoval() + { + var manager = CreateTestSystem().Manager; + var hook = new DelegateTestHook(); + await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); + + await manager.DisposeAsync(); + + await Assert.ThrowsAsync(() => manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + } + + [Fact] + public async Task Hooks_RegistrationDuringRecoveryAppliesBeforeLaterPersistence() + { + var entered = NewSignal(); + var release = NewSignal(); + var storage = Substitute.For(); + storage.ReadAsync(Arg.Any(), Arg.Any()) + .Returns(call => ReadAsync(call.Arg(), call.Arg())); + await using var manager = CreateTestSystem(storage).Manager; + var before = 0; + var after = 0; + var hook = new DelegateTestHook + { + BeforeOperation = (_, _) => before++, + AfterOperation = (_, _) => after++ + }; + var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var registration = manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.False(initialization.IsCompleted); + Assert.False(registration.IsCompleted); + release.TrySetResult(); + await WaitFor(Task.WhenAll(initialization, registration)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + } + finally + { + release.TrySetResult(); + } + + async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + consumer.Complete(metadata: null); + } + } + + [Fact] + public async Task Hooks_QueuedRegistrationReportsRecoveryFailureAndIdleRegistrationDoesNotRetryRecovery() + { + var entered = NewSignal(); + var release = NewSignal(); + var storage = Substitute.For(); + var failure = new IOException("Recovery failed."); + var fail = true; + var reads = 0; + storage.ReadAsync(Arg.Any(), Arg.Any()) + .Returns(call => ReadAsync(call.Arg(), call.Arg())); + await using var manager = CreateTestSystem(storage).Manager; + var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var failedBefore = 0; + var failedRegistration = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => failedBefore++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + release.TrySetResult(); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(failedRegistration))); + var before = 0; + var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; + Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + Assert.Equal(1, reads); + fail = false; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, reads); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, failedBefore); + Assert.Equal(1, before); + } + finally + { + release.TrySetResult(); + } + + async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) + { + reads++; + entered.TrySetResult(); + await release.Task.WaitAsync(token); + if (fail) + { + throw failure; + } + consumer.Complete(metadata: null); + } + } + + [Theory] + [InlineData("before")] + [InlineData("storage")] + [InlineData("after")] + public async Task Hooks_BusyMutationWaitsForCompleteOperationAndSurvivesCallerCancellation(string phase) + { + var storage = new CapturingStorage { BlockNextAppend = phase == "storage" }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var oldBefore = 0; + var oldAfter = 0; + var newBefore = 0; + var newAfter = 0; + var original = new DelegateTestHook + { + BeforeOperationAsync = async (_, _) => + { + oldBefore++; + if (phase == "before") + { + entered.TrySetResult(); + await release.Task; + } + }, + AfterOperationAsync = async (_, _) => + { + oldAfter++; + if (phase == "after") + { + entered.TrySetResult(); + await release.Task; + } + } + }; + var replacement = new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }; + await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(phase == "storage" ? storage.BlockedAppendStarted.Task : entered.Task); + using var caller = new CancellationTokenSource(); + var added = manager.AddHookAsync(replacement, index: 0, caller.Token).AsTask(); + var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + state.Value = 2; + var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(added.IsCompleted); + Assert.False(removed.IsCompleted); + Assert.Equal(0, newBefore); + Assert.Equal(0, newAfter); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => added); + + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + await WaitFor(Task.WhenAll(first, removed, next)); + Assert.Equal(1, oldBefore); + Assert.Equal(1, oldAfter); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + Assert.Equal(2, state.Value); + } + finally + { + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_QueuedMutationSeparatesWriteCoalescingGroups() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var oldBefore = 0; + var oldAfter = 0; + var newBefore = 0; + var newAfter = 0; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => oldBefore++, + AfterOperation = (_, _) => oldAfter++ + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var beforeChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var change = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var afterChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var coalesced = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await WaitFor(Task.WhenAll(first, beforeChange, change, afterChange, coalesced)); + Assert.Equal(3, state.CaptureCount); + Assert.Equal(3, storage.Appends.Count); + Assert.Equal(3, oldBefore); + Assert.Equal(3, oldAfter); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ResetCanQueueRegistrationWithoutChangingCurrentAfterHooks() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + Task? registration = null; + var newBefore = 0; + var newAfter = 0; + state.ResetAction = () => + { + if (state.ResetCount == 2) + { + registration = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + } + }; + manager.RegisterStateMachine("state", state); + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperation = (operation, _) => + { + if (operation == JournaledStateOperation.Delete) + { + Assert.NotNull(registration); + Assert.False(registration.IsCompleted); + Assert.Equal(0, newBefore); + Assert.Equal(0, newAfter); + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.DeleteStateAsync(TestContext.Current.CancellationToken); + Assert.NotNull(registration); + await WaitFor(registration); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + } + + [Fact] + public async Task Hooks_FailedQueuedMutationReportsErrorAndLeavesOwnerUsable() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var invalid = manager.AddHookAsync(new DelegateTestHook(), index: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var before = 0; + var valid = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; + var accepted = manager.AddHookAsync(valid, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await Assert.ThrowsAsync(() => WaitFor(invalid)); + await WaitFor(Task.WhenAll(first, accepted)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_StorageFailureFaultsQueuedRegistrationWithOriginalCause() + { + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage { BlockNextAppend = true, NextAppendException = failure }; + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var registration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(write))); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(registration))); + var fenced = await Assert.ThrowsAsync(() => + manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, fenced.InnerException); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ShutdownCancelsQueuedMutationWithoutChangingCommittedOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + var entered = NewSignal(); + var release = NewSignal(); + var original = new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } + }; + await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var added = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.False(shutdown.IsCompleted); + release.TrySetResult(); + await WaitFor(Task.WhenAll(write, shutdown)); + await Assert.ThrowsAnyAsync(() => WaitFor(added)); + await Assert.ThrowsAnyAsync(() => WaitFor(removed)); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot, BlockNextAppend = !snapshot, BlockNextReplace = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; + CancellationToken ownedToken = default; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.True(token.CanBeCanceled); + ownedToken = token; + events.Add("before-sync"); + state["hook"] = 2; + }, + BeforeOperationAsync = async (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("before-async"); + }, + AfterOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + events.Add("after-sync"); + }, + AfterOperationAsync = async (_, token) => + { + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("after-async"); + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await (snapshot ? storage.ReplaceEntered : storage.BlockedAppendStarted).Task + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture" }, events); + Assert.False(write.IsCompleted); + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture", "ack", "after-sync", "after-async" }, events); + } + finally + { + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + } + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["hook"]); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var capture = new AlwaysWritingState(); + manager.RegisterStateMachine("capture", capture); + var failure = new IOException("Prerequisite failed."); + var fail = true; + var after = 0; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => + { + if (fail) + { + throw failure; + } + }, + AfterOperation = (_, _) => after++ + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var pendingBytes = manager.PendingWriteByteCount; + var caught = await Assert.ThrowsAsync(() => InvokeAsync()); + Assert.Same(failure, caught.InnerException); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); + Assert.Equal(1, state["business"]); + Assert.Equal(pendingBytes, manager.PendingWriteByteCount); + Assert.Equal(0, capture.AppendEntriesCount); + Assert.Equal(0, capture.WriteCompletedCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + storage.IsCompactionRequested = false; + fail = false; + await InvokeAsync(); + Assert.Equal(1, after); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); + Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); + + Task InvokeAsync() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask(); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHooks(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + var fail = true; + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperation = (_, _) => + { + if (fail) + { + throw first; + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + await Task.Yield(); + events.Add("second"); + if (fail) + { + throw second; + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + Assert.Equal(operation == JournaledStateOperation.Delete + ? new[] { "second", "last" } + : new[] { "capture", "ack", "second", "last" }, events); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + fail = false; + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, recovered.Count); + if (operation != JournaledStateOperation.Delete) + { + Assert.Equal(1, recovered["business"]); + } + + state["later"] = 2; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, state["later"]); + } + + [Fact] + public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() + { + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage { NextAppendException = failure }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var after = 0; + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + Assert.Equal(0, after); + var fenced = await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, fenced.InnerException); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinaryHook(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await manager.AddHookAsync(new DelegateTestHook( + (_, _) => + { + Assert.Equal(3, state["late"]); + state["owner"] = 2; + events.Add("capture-prerequisite"); + return default; + }, + (_, _) => events.Add("capture-after")), cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperationAsync = async (_, token) => + { + events.Add("ordinary-before"); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + events.Add("ordinary-complete"); + }, + AfterOperation = (_, _) => events.Add("ordinary-after") + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before" }, events); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + state["late"] = 3; + release.TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before", "ordinary-complete", "capture-prerequisite", "capture", "ack", "capture-after", "ordinary-after" }, events); + } + finally + { + release.TrySetResult(); + } + + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["owner"]); + Assert.Equal(3, recovered["late"]); + } + + [Fact] + public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken ownedToken = default; + Func before = async (_, token) => + { + ownedToken = token; + entered.TrySetResult(); + await release.Task.WaitAsync(token); + state["scheduled"] = 2; + }; + Action after = (_, token) => + { + Assert.Equal(ownedToken, token); + Assert.False(token.IsCancellationRequested); + Assert.Single(storage.Appends); + completed.TrySetResult(); + }; + await manager.AddHookAsync(new DelegateTestHook(before, after), cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + using var caller = new CancellationTokenSource(); + state["business"] = 1; + var write = manager.WriteStateAsync(caller.Token).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.NotEqual(caller.Token, ownedToken); + state["late"] = 3; + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.False(ownedToken.IsCancellationRequested); + Assert.Empty(storage.Appends); + Assert.False(completed.Task.IsCompleted); + release.TrySetResult(); + await completed.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, state["scheduled"]); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["scheduled"]); + Assert.Equal(3, recovered["late"]); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_SynchronousFinalPrerequisiteAndCaptureShareContinuationAfterAsyncEarlierHook(bool snapshot) + { + var scheduler = new QueuedHookTaskScheduler(); + await scheduler.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + List prepared = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + var queueMutation = true; + Task? mutation = null; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => + { + events.Add("final-before"); + prepared.AddRange(state.Keys); + if (queueMutation) + { + Assert.Empty(state); + queueMutation = false; + mutation = scheduler.Queue(() => + { + state["intent"] = 1; + events.Add("queued-mutation"); + }); + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperationAsync = async (_, _) => + { + events.Add("ordinary-before"); + await Task.Yield(); + events.Add("ordinary-complete"); + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(mutation); + await mutation; + Assert.Empty(prepared); + Assert.Equal(1, state["intent"]); + await using (var recoveredManager = CreateTestSystem(storage).Manager) + { + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Empty(recovered); + } + + Assert.Equal( + ["ordinary-before", "ordinary-complete", "final-before", "capture", "ack", "queued-mutation"], + events); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(["intent"], prepared); + await using var replayManager = CreateTestSystem(storage).Manager; + var replayed = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + var intent = Assert.Single(replayed); + Assert.Equal("intent", intent.Key); + Assert.Equal(1, intent.Value); + }); + } + + [Fact] + public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrder() + { + await using var manager = CreateTestSystem().Manager; + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("second-before"), + AfterOperation = (_, _) => events.Add("second-after") + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("third-before"), + AfterOperation = (_, _) => events.Add("third-after") + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("first-before"), + AfterOperation = (_, _) => events.Add("first-after") + }, index: 0, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal( + ["third-before", "second-before", "first-before", "capture", "ack", "first-after", "second-after", "third-after"], + events); + } + + [Fact] + public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var deleting = false; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var before = 0; + var after = 0; + await manager.AddHookAsync(new DelegateTestHook( + async (operation, token) => + { + before++; + if (!deleting) + { + Assert.Equal(JournaledStateOperation.Write, operation); + return; + } + + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(1, state.Value); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + }, + (operation, _) => + { + if (!deleting) + { + return; + } + + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(0, state.Value); + Assert.Equal(1, storage.DeleteCount); + after++; + }), cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + deleting = true; + var deletion = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + Assert.False(deletion.IsCompleted); + release.TrySetResult(); + await deletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, after); + deleting = false; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, before); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var drained = false; + var failure = new IOException("Cancellation callback failed."); + var hook = new DelegateTestHook + { + BeforeOperationAsync = async (_, token) => + { + using var registration = token.Register(() => throw failure); + entered.TrySetResult(); + await release.Task; + drained = true; + } + }; + await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + Assert.False(drained); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Same(failure, Assert.Single(caught.InnerExceptions)); + await Assert.ThrowsAnyAsync(() => write); + await Assert.ThrowsAnyAsync(() => queued); + Assert.True(drained); + Assert.Empty(storage.Appends); + await Assert.ThrowsAsync(() => + manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + } + finally + { + release.TrySetResult(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDrains(bool after) + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var entered = NewSignal(); + var release = NewSignal(); + Func callback = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + }; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperationAsync = after ? null : callback, + AfterOperationAsync = after ? callback : null + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var initializing = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var deleting = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.False(shutdown.IsCompleted); + Assert.False(initializing.IsCompleted); + Assert.False(deleting.IsCompleted); + release.TrySetResult(); + await WaitFor(shutdown); + if (after) + { + await WaitFor(write); + } + else + { + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + } + + await Assert.ThrowsAnyAsync(() => WaitFor(initializing)); + await Assert.ThrowsAnyAsync(() => WaitFor(deleting)); + Assert.True(initializing.IsCanceled); + Assert.True(deleting.IsCanceled); + Assert.Equal(after ? 1 : 0, state.CaptureCount); + Assert.Equal(after ? 1 : 0, state.WriteCompletedCount); + Assert.Equal(after ? 1 : 0, storage.Appends.Count); + Assert.Equal(0, storage.DeleteCount); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesItsOutcome(bool snapshot, bool storageFails) + { + var entered = NewSignal(); + var release = NewSignal(); + var cancellationFailure = new IOException("Cancellation callback failed."); + var storageFailure = new IOException("Actual storage failure."); + var completed = false; + var storage = Substitute.For(); + storage.IsCompactionRequested.Returns(snapshot); + storage.ReadAsync(Arg.Any(), Arg.Any()).Returns(call => + { + call.Arg().Complete(metadata: null); + return ValueTask.CompletedTask; + }); + storage.AppendAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + storage.ReplaceAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + CancellationTokenRegistration registration = default; + var after = 0; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), + AfterOperation = (_, _) => after++ + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, manager.DisposeAsync().AsTask()); + Assert.False(shutdown.IsCompleted); + Assert.False(completed); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Equal(0, after); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => WaitFor(shutdown)); + Assert.Same(cancellationFailure, Assert.Single(caught.InnerExceptions)); + Assert.True(completed); + if (storageFails) + { + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(write))); + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(queued))); + } + else + { + await WaitFor(write); + await Assert.ThrowsAnyAsync(() => WaitFor(queued)); + } + + Assert.Equal(storageFails ? 0 : 1, state.WriteCompletedCount); + Assert.Equal(storageFails ? 0 : 1, after); + } + finally + { + release.TrySetResult(); + registration.Dispose(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + + async ValueTask WriteAsync(ReadOnlySequence bytes, CancellationToken token) + { + var original = bytes.ToArray(); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + Assert.Equal(original, bytes.ToArray()); + completed = true; + if (storageFails) + { + throw storageFailure; + } + } + } + + [Fact] + public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + Task? reenteredDisposal = null; + var callbackCount = 0; + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperationAsync = async (_, token) => + { + using var registration = token.Register(() => + { + callbackCount++; + reenteredDisposal = manager.DisposeAsync().AsTask(); + }); + entered.TrySetResult(); + await release.Task; + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var disposal = manager.DisposeAsync().AsTask(); + Assert.Equal(1, callbackCount); + Assert.Same(disposal, reenteredDisposal); + Assert.False(disposal.IsCompleted); + release.TrySetResult(); + await WaitFor(Task.WhenAll(write, disposal)); + Assert.Same(disposal, manager.DisposeAsync().AsTask()); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + List events = []; + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperationAsync = async (_, token) => + { + Assert.Single(storage.Appends); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + events.Add("first"); + throw first; + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => throw second }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "first", "last" }, events); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public async Task Hooks_QueuedWritesCoalesceAndRetainSnapshotRequirement() + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + List before = []; + List after = []; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (operation, _) => before.Add(operation), + AfterOperation = (operation, _) => after.Add(operation) + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var append = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = true; + state.Value = 2; + var snapshot = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = false; + state.Value = 3; + var anotherAppend = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Empty(before); + await Task.WhenAll(append, snapshot, anotherAppend); + Assert.Equal([JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Single(storage.Replaces); + + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Value); + }); + } + + [Fact] + public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() + { + var storage = new CapturingStorage { IsCompactionRequested = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var failure = new IOException("Snapshot prerequisite failed."); + List before = []; + List after = []; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperationAsync = async (operation, _) => + { + before.Add(operation); + if (before.Count == 1) + { + entered.TrySetResult(); + await release.Task; + throw failure; + } + }, + AfterOperation = (operation, _) => after.Add(operation) + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + storage.IsCompactionRequested = false; + state.Value = 2; + var retry = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => WaitFor(first)); + Assert.Same(failure, caught.InnerException); + await WaitFor(retry); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Single(storage.Replaces); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Value); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() + { + var storage = new CapturingStorage { IsCompactionRequested = true, BlockNextReplace = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + List before = []; + List after = []; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (operation, _) => before.Add(operation), + AfterOperation = (operation, _) => after.Add(operation) + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.ReplaceEntered.Task); + state.Value = 2; + var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = false; + Assert.Empty(storage.Replaces); + Assert.Equal([JournaledStateOperation.Snapshot], before); + Assert.Empty(after); + storage.ReleaseReplace.TrySetResult(); + await WaitFor(Task.WhenAll(first, second)); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Equal(2, storage.Replaces.Count); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Value); + } + finally + { + storage.ReleaseReplace.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var before = 0; + var after = 0; + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["first"] = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await storage.BlockedAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(0, after); + state["second"] = 2; + var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + state["third"] = 3; + var third = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, before); + Assert.Equal(2, after); + Assert.Equal(2, storage.Appends.Count); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var after = 0; + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + if (++after == 1) + { + entered.TrySetResult(); + await release.Task; + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["first"] = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.Single(storage.Appends); + Assert.False(first.IsCompleted); + state["later"] = 2; + var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(next.IsCompleted); + + await using (var beforeNext = CreateTestSystem(storage).Manager) + { + var recovered = new DurableDictionary("state", beforeNext, CreateDictionaryCodec()); + await beforeNext.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(recovered).Value); + } + + release.TrySetResult(); + await WaitFor(Task.WhenAll(first, next)); + Assert.Equal(2, after); + Assert.Equal(2, storage.Appends.Count); + await using var afterNext = CreateTestSystem(storage).Manager; + var persisted = new DurableDictionary("state", afterNext, CreateDictionaryCodec()); + await afterNext.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, persisted.Count); + Assert.Equal(1, persisted["first"]); + Assert.Equal(2, persisted["later"]); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var before = 0; + var after = 0; + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + before = 0; + after = 0; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + Assert.Single(storage.Appends); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) + { + Exception original = after + ? new JournaledStatePostCommitException(JournaledStateOperation.Delete, new IOException("Cleanup failed.")) + : new JournaledStatePreCommitException(JournaledStateOperation.Snapshot, new IOException("Scheduling failed.")); + var serializer = ServiceProvider.GetRequiredService(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + if (after) + { + Assert.IsType(copy); + } + else + { + Assert.IsType(copy); + } + + Assert.Equal(original.InnerException!.Message, Assert.IsType(copy.InnerException).Message); + Assert.Equal(original.Message, copy.Message); + } + + private sealed class QueuedHookTaskScheduler : TaskScheduler + { + private readonly Channel _queue = Channel.CreateUnbounded(); + + public override int MaximumConcurrencyLevel => 1; + protected override IEnumerable? GetScheduledTasks() => null; + protected override void QueueTask(Task task) => Assert.True(_queue.Writer.TryWrite(task)); + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; + + public Task Queue(Action action) => + Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this); + + public async Task Run(Func action) + { + var task = Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this).Unwrap(); + while (!task.IsCompleted) + { + var next = await _queue.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(null); + try + { + Assert.True(TryExecuteTask(next)); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + } + } + + await task; + } + } + + private sealed class DelegateTestHook( + Func? before = null, + Action? after = null) : IJournaledStateHook + { + public Action? BeforeOperation { get; init; } + public Func? BeforeOperationAsync { get; init; } = before; + public Action? AfterOperation { get; init; } = after; + public Func? AfterOperationAsync { get; init; } + + ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + BeforeOperation?.Invoke(operation, cancellationToken); + return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + + ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + AfterOperation?.Invoke(operation, cancellationToken); + return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + } + + private sealed class HookAcknowledgementState(List events) : IStateMachine + { + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } + public void Reset(JournalStreamWriter writer) { } + public void WritePendingEntries(JournalStreamWriter writer) => events.Add("capture"); + public void WriteSnapshot(JournalStreamWriter writer) => events.Add("capture"); + public void OnWriteCompleted() => events.Add("ack"); + } +} diff --git a/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs new file mode 100644 index 00000000000..742bc1a2e77 --- /dev/null +++ b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs @@ -0,0 +1,282 @@ +using Xunit; + +namespace Orleans.Journaling.Tests; + +public partial class StateManagerTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_RunsAfterHooksBeforeCaptureAndAcknowledgement(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + List events = []; + var state = new PreparingValue("state", manager, CreateValueCodec(), events); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => { events.Add("before"); state["intent"] = 17; }, + AfterOperation = (_, _) => events.Add("after") + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["before", "prepare", "capture", "ack", "after"], events); + Assert.Equal(17, state.CapturedValue); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_LaterAsynchronousStateRechecksEarlierInterleavedChanges(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var earlier = new PreparingValue("earlier", manager, CreateValueCodec()); + var later = new PreparingValue("later", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var prepared = false; + later.Preparation = async token => + { + if (!prepared) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + later.PreparedValue = later.CurrentValue; + prepared = true; + } + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + later["intent"] = 23; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.Equal(0, earlier.CaptureCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + earlier["intent"] = 41; + release.TrySetResult(); + await WaitFor(write); + Assert.Equal(41, earlier.CapturedValue); + Assert.Equal(41, earlier.PreparedValue); + Assert.Equal(2, earlier.PrepareCalls); + Assert.Equal(23, later.CapturedValue); + Assert.Equal(1, earlier.WriteCompletedCount); + + await using var recovered = CreateTestSystem(storage).Manager; + var replayed = new PreparingValue("earlier", recovered, CreateValueCodec()); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(41, replayed["intent"]); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_SynchronousReadyPassAndCaptureShareOneTurn(bool snapshot) + { + var scheduler = new QueuedHookTaskScheduler(); + await scheduler.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new PreparingValue("state", manager, CreateValueCodec()); + Task? mutation = null; + state.Preparation = _ => + { + state.PreparedValue = state.CurrentValue; + mutation ??= scheduler.Queue(() => state["intent"] = 29); + return ValueTask.CompletedTask; + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 13; + + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + await mutation!; + + Assert.Equal(13, state.CapturedValue); + Assert.Equal(29, state.CurrentValue); + await using var recovered = CreateTestSystem(storage).Manager; + var replayed = new PreparingValue("state", recovered, CreateValueCodec()); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(13, replayed["intent"]); + }); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Preparation_FailureRetainsPendingStateAndSnapshotForExplicitRetry(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var failure = new IOException("State prerequisite failed."); + var state = new PreparingValue("state", manager, CreateValueCodec()) + { + Preparation = _ => ValueTask.FromException(failure) + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 31; + + var caught = await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + + Assert.Same(failure, caught.InnerException); + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + storage.IsCompactionRequested = false; + state.Preparation = null; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(31, state.CapturedValue); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + Assert.Empty(snapshot ? storage.Appends : storage.Replaces); + } + + [Fact] + public async Task Preparation_CallerCancellationLeavesOwnedPreparationAndWriteRunning() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new PreparingValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var prepared = false; + state.Preparation = async token => + { + if (!prepared) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + state.PreparedValue = state.CurrentValue; + prepared = true; + } + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 47; + using var caller = new CancellationTokenSource(); + var write = manager.WriteStateAsync(caller.Token).AsTask(); + try + { + await WaitFor(entered.Task); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + Assert.Equal(0, state.CaptureCount); + release.TrySetResult(); + await WaitFor(state.Acknowledged.Task); + Assert.Equal(47, state.CapturedValue); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task Preparation_ShutdownDrainsPreparationAndPreventsCapture() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new PreparingValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var observedCancellation = false; + state.Preparation = async token => + { + entered.TrySetResult(); + await release.Task; + observedCancellation = token.IsCancellationRequested; + state.PreparedValue = state.CurrentValue; + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 53; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var disposal = manager.DisposeAsync().AsTask(); + Assert.False(disposal.IsCompleted); + release.TrySetResult(); + await WaitFor(disposal); + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + Assert.True(observedCancellation); + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Empty(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + private sealed class PreparingValue : IStateMachine, IDurableValueCommandHandler + { + private readonly IDurableValueCommandCodec _codec; + private readonly List? _events; + private JournalStreamWriter _writer; + + public PreparingValue(string name, IJournaledStateManager manager, IDurableValueCommandCodec codec, List? events = null) + { + _codec = codec; + _events = events; + manager.RegisterStateMachine(name, this); + } + public Func? Preparation { get; set; } + public int PrepareCalls { get; private set; } + public int PreparedValue { get; set; } + public int CapturedValue { get; private set; } + public int CaptureCount { get; private set; } + public int WriteCompletedCount { get; private set; } + private int _value; + public int CurrentValue => _value; + public int this[string key] + { + get => _value; + set { _codec.WriteSet(value, _writer); _value = value; } + } + void IDurableValueCommandHandler.ApplySet(int command) => _value = command; + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); + public void Reset(JournalStreamWriter writer) { _writer = writer; _value = 0; } + public void WritePendingEntries(JournalStreamWriter writer) => Capture(); + public void WriteSnapshot(JournalStreamWriter writer) { _codec.WriteSet(_value, writer); Capture(); } + public TaskCompletionSource Acknowledged { get; } = NewSignal(); + + public ValueTask PrepareWriteAsync(CancellationToken cancellationToken = default) + { + PrepareCalls++; + _events?.Add("prepare"); + if (Preparation is { } preparation) return preparation(cancellationToken); + PreparedValue = CurrentValue; + return ValueTask.CompletedTask; + } + + private void Capture() + { + _events?.Add("capture"); + CaptureCount++; + CapturedValue = CurrentValue; + Assert.Equal(PreparedValue, CapturedValue); + } + + public void OnWriteCompleted() + { + _events?.Add("ack"); + WriteCompletedCount++; + Acknowledged.TrySetResult(); + } + } +} From 5e10ce5f02cefa59cde923481fb99ab23454d9fb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 14:45:00 -0700 Subject: [PATCH 28/95] feat(messaging): process inbox messages Adds the durable inbox, its journaled state, and the per-activation messaging pump which processes accepted messages on durable-job wake-ups. AddMessaging registers the inbox, inbox dead-letter diagnostics, and the grain-type configurator which installs the inbox extension during activation. --- src/Orleans.Core/Orleans.Core.csproj | 1 + src/Orleans.Messaging/DeadLetterRetention.cs | 41 + .../Hosting/MessagingExtensions.cs | 85 ++ .../IMessagingDiagnostics.cs | 72 ++ src/Orleans.Messaging/IMessagingGrain.cs | 14 + src/Orleans.Messaging/Inbox.State.cs | 183 +++++ src/Orleans.Messaging/Inbox.cs | 427 ++++++++++ src/Orleans.Messaging/InboxHandlerContext.cs | 10 + src/Orleans.Messaging/MessageState.cs | 33 + .../MessagingActivationValidator.cs | 20 + .../MessagingGrainTypeConfigurator.cs | 31 + src/Orleans.Messaging/MessagingInstruments.cs | 64 ++ src/Orleans.Messaging/MessagingPump.cs | 231 ++++++ src/Orleans.Messaging/MessagingShared.cs | 55 ++ src/Orleans.Messaging/MessagingStateNames.cs | 8 + src/Orleans.Messaging/MessagingTime.cs | 21 + .../Orleans.Messaging.csproj | 3 + src/Orleans.Messaging/README.md | 64 +- .../Orleans.Messaging/Orleans.Messaging.cs | 29 + .../Contracts/ActivationValidationTests.cs | 99 +++ .../DeliveryAndOptionsContractTests.cs | 31 + .../Contracts/EnvelopeContractTests.cs | 1 + .../MessagingCommandSerializationTests.cs | 65 ++ .../Contracts/StandardJournaledStateTests.cs | 105 +++ .../Functional/ByteMessagingPayloadTests.cs | 109 +++ .../Functional/DedupeExpiryBehaviorTests.cs | 107 +++ .../EarlyActivationValidationTests.cs | 82 ++ .../InboxAcceptanceBehaviorTests.cs | 171 ++++ .../Functional/InboxAdmissionTests.cs | 295 +++++++ .../Functional/InboxCapacityBehaviorTests.cs | 80 ++ .../InboxEnvelopeValidationTests.cs | 73 ++ .../Functional/InboxHandlerCompletionTests.cs | 731 ++++++++++++++++++ .../InboxHandlerTransactionTests.cs | 154 ++++ .../Functional/InboxHeaderBehaviorTests.cs | 260 +++++++ .../Functional/InboxIdentityTests.cs | 462 +++++++++++ .../Functional/InboxJournalRecoveryTests.cs | 96 +++ .../Functional/InboxLifecycleBehaviorTests.cs | 57 ++ .../Functional/InboxQuiescenceTests.cs | 92 +++ .../Functional/InboxRetryTests.cs | 134 ++++ .../Functional/InboxStateEncodingTests.cs | 99 +++ .../InboxStateManagerBoundaryTests.cs | 228 ++++++ .../Functional/InboxStateProtocolTests.cs | 98 +++ .../JournaledTestOutboxBehaviorTests.cs | 122 +++ .../Functional/MessagingClusterCollection.cs | 9 + .../MessagingMetricCardinalityTests.cs | 262 +++++++ .../MessagingRoutingAndDeadLetterTests.cs | 147 ++++ .../ProcessedRetentionMaintenanceTests.cs | 65 ++ .../Functional/StandardStateJsonContext.cs | 9 + .../PublicMessagingRegistrationTests.cs | 144 ++++ .../Orleans.Messaging.Tests.csproj | 2 + .../Support/ActivationValidationTestGrains.cs | 137 ++++ .../Support/ControlledDurableJobManager.cs | 151 ++++ .../ControlledJournalStorageProvider.cs | 282 +++++++ .../Support/FaultingInboxCodecFixture.cs | 38 + .../Support/HandlerProbe.cs | 53 ++ .../Support/InboxTestAccess.cs | 44 ++ .../Support/JournaledTestOutbox.cs | 63 ++ .../Support/MessagingBehaviorTestBase.cs | 83 ++ .../Support/MessagingClusterFixture.cs | 272 +++++++ .../Support/MessagingMetricProbe.cs | 121 +++ .../Support/MessagingTestGrains.cs | 469 +++++++++++ .../Support/NamedFactoryMessagingGrain.cs | 111 +++ .../Support/ReceiverTestServices.cs | 27 + .../Support/SnapshotProbe.cs | 98 +++ .../Support/SnapshotProbeTests.cs | 82 ++ .../Support/TestApplicationProtocol.cs | 103 +++ 66 files changed, 7904 insertions(+), 11 deletions(-) create mode 100644 src/Orleans.Messaging/DeadLetterRetention.cs create mode 100644 src/Orleans.Messaging/Hosting/MessagingExtensions.cs create mode 100644 src/Orleans.Messaging/IMessagingDiagnostics.cs create mode 100644 src/Orleans.Messaging/IMessagingGrain.cs create mode 100644 src/Orleans.Messaging/Inbox.State.cs create mode 100644 src/Orleans.Messaging/Inbox.cs create mode 100644 src/Orleans.Messaging/InboxHandlerContext.cs create mode 100644 src/Orleans.Messaging/MessageState.cs create mode 100644 src/Orleans.Messaging/MessagingActivationValidator.cs create mode 100644 src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs create mode 100644 src/Orleans.Messaging/MessagingInstruments.cs create mode 100644 src/Orleans.Messaging/MessagingPump.cs create mode 100644 src/Orleans.Messaging/MessagingShared.cs create mode 100644 src/Orleans.Messaging/MessagingStateNames.cs create mode 100644 src/Orleans.Messaging/MessagingTime.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs create mode 100644 test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs create mode 100644 test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs create mode 100644 test/Orleans.Messaging.Tests/Support/HandlerProbe.cs create mode 100644 test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs create mode 100644 test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs create mode 100644 test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs create mode 100644 test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs create mode 100644 test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs create mode 100644 test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs diff --git a/src/Orleans.Core/Orleans.Core.csproj b/src/Orleans.Core/Orleans.Core.csproj index 411bc41ff8e..b73ca206771 100644 --- a/src/Orleans.Core/Orleans.Core.csproj +++ b/src/Orleans.Core/Orleans.Core.csproj @@ -76,6 +76,7 @@ + diff --git a/src/Orleans.Messaging/DeadLetterRetention.cs b/src/Orleans.Messaging/DeadLetterRetention.cs new file mode 100644 index 00000000000..e2388a608e9 --- /dev/null +++ b/src/Orleans.Messaging/DeadLetterRetention.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Orleans.Messaging; + +internal static class DeadLetterRetention +{ + public static bool Compact( + IDictionary entries, + DateTimeOffset now, + TimeSpan retentionPeriod, + int maxRetainedEntries, + Func getTimestamp, + int reservedCapacity = 0) + where TKey : notnull + { + var removed = false; + foreach (var entry in entries + .Where(entry => MessagingTime.IsExpired(now, getTimestamp(entry.Value), retentionPeriod)) + .ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + var removeCount = entries.Count + reservedCapacity - maxRetainedEntries; + if (removeCount <= 0) + { + return removed; + } + + foreach (var entry in entries.OrderBy(entry => getTimestamp(entry.Value)).Take(removeCount).ToList()) + { + entries.Remove(entry.Key); + removed = true; + } + + return removed; + } +} diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs new file mode 100644 index 00000000000..d0cd1978e82 --- /dev/null +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -0,0 +1,85 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Messaging; +using Orleans.Messaging.Configuration; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Journaling.Json; + +namespace Orleans.Hosting; + +/// +/// Extensions for configuring messaging. +/// +public static class MessagingExtensions +{ + /// + /// Adds inbox messaging support to the silo. + /// + /// + /// Grains implementing or deriving from + /// bind their scoped messaging services during activation setup, before journal recovery. + /// + /// The silo builder. + /// The optional action used to configure inbox and outbox behavior. + /// The silo builder. + public static ISiloBuilder AddMessaging(this ISiloBuilder builder, Action? configureOptions = null) + { + builder.AddDurableJobs(); + return builder.ConfigureServices(services => services.AddMessaging(configureOptions)); + } + + /// + /// Adds inbox messaging services. + /// + /// + /// Grains implementing or deriving from + /// bind their scoped messaging services during activation setup, before journal recovery. + /// + /// The service collection. + /// The optional action used to configure inbox and outbox behavior. + /// The service collection. + public static IServiceCollection AddMessaging(this IServiceCollection services, Action? configureOptions = null) + { + services.AddDurableJobs(); + services.TryAddSingleton(TimeProvider.System); + services.Configure(options => + { + options.AddTypeInfoResolver(InboxJsonContext.Default); + }); + + var optionsBuilder = services.AddOptions(); + if (configureOptions is not null) + { + optionsBuilder.Configure(configureOptions); + } + + optionsBuilder.Validate( + options => + { + try + { + options.Validate(); + return true; + } + catch (ArgumentOutOfRangeException) + { + return false; + } + }, + "DurableMessagingOptions validation failed."); + + services.ConfigureNamedOptionForLogging(Options.DefaultName); + services.TryAddSingleton(); + services.TryAddScoped(); + services.TryAddScoped(); + services.TryAddScoped(static sp => sp.GetRequiredService()); + services.TryAddScoped(); + services.TryAddEnumerable( + ServiceDescriptor.Singleton()); + return services; + } +} diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs new file mode 100644 index 00000000000..ea28b65e64d --- /dev/null +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Provides operational access to a grain's messaging state. +/// +public interface IMessagingDiagnostics +{ + /// + /// Gets messages which failed during inbox processing. + /// + IReadOnlyList> InboxDeadLetters { get; } + + /// + /// Stages removal of an inbox dead letter. + /// + /// The receiver-local command identity. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveInboxDeadLetter(HierarchicalKey messageId); +} + +/// +/// Describes a dead-lettered durable message. +/// +/// The incoming or outgoing message type. +public sealed class DeadLetter +{ + /// + /// Gets the message. + /// + /// + /// Treat the message's encoded headers as immutable. Application diagnostic DTOs use + /// ordinary RPC copying when returning message buffers. + /// + public required TMessage Message { get; init; } + + /// + /// Gets when the message was dead-lettered. + /// + public DateTimeOffset DeadLetteredAt { get; init; } + + /// + /// Gets the terminal failure reason. + /// + public required string Reason { get; init; } + + /// + /// Gets the number of attempts made. + /// + public int AttemptCount { get; init; } +} + +internal sealed class MessagingDiagnostics(Inbox inbox) : IMessagingDiagnostics +{ + public IReadOnlyList> InboxDeadLetters => + inbox.DeadLetterState.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + + public bool RemoveInboxDeadLetter(HierarchicalKey messageId) => inbox.RemoveDeadLetter(messageId); +} diff --git a/src/Orleans.Messaging/IMessagingGrain.cs b/src/Orleans.Messaging/IMessagingGrain.cs new file mode 100644 index 00000000000..718d6952c0f --- /dev/null +++ b/src/Orleans.Messaging/IMessagingGrain.cs @@ -0,0 +1,14 @@ +namespace Orleans.Messaging; + +/// +/// Identifies grain implementations which initialize inbox and outbox services during activation setup. +/// +/// +/// Implement this local capability on a grain class, an application base class, or an application grain interface. +/// With messaging services registered, selected activations validate their execution model and bind +/// messaging state before journal recovery. Grains deriving from +/// receive the same setup automatically. +/// +public interface IMessagingGrain +{ +} diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs new file mode 100644 index 00000000000..e496b3adc0f --- /dev/null +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Journaling; + +namespace Orleans.Messaging; + +internal sealed partial class Inbox +{ + private readonly Dictionary _messages = []; + private readonly Dictionary _processed = []; + private readonly Dictionary _deadLetters = []; + private JournalStreamWriter _writer; + + // Entries are durable once a completed write captured their sequence number. + private long _lastSequence; + private long _capturedSequence; + private long _durableSequence; + + private bool IsDurable(InboxEntry entry) => entry.Sequence <= _durableSequence; + + ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) => + _lastSequence != _capturedSequence ? _pump.EnsureWakeupAsync(cancellationToken) : ValueTask.CompletedTask; + + private void OnWriteCaptured() => _capturedSequence = _lastSequence; + + void IStateMachine.OnWriteCompleted() + { + _durableSequence = _capturedSequence; + _pump.Signal(); + } + + private void Emit(InboxCommand command) + { + Shared.InboxCodec.WriteSet(command, _writer); + Apply(command); + } + + void IStateMachine.Reset(JournalStreamWriter writer) + { + ReportDepth(-_messages.Count); + _writer = writer; + _messages.Clear(); + _processed.Clear(); + _deadLetters.Clear(); + _lastSequence = _capturedSequence = _durableSequence = 0; + } + + void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, Shared.InboxCodec).Apply(entry.Reader, this); + + void IStateMachine.OnRecoveryCompleted() + { + foreach (var (key, entry) in _messages) + { + EnvelopeValidation.Validate(entry.Message); + if (key != entry.Message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered inbox key does not match its command identity."); + } + } + + foreach (var letter in _deadLetters.Values) EnvelopeValidation.Validate(letter.Message); + _capturedSequence = _durableSequence = _lastSequence; + } + + void IStateMachine.WritePendingEntries(JournalStreamWriter writer) => OnWriteCaptured(); + + void IStateMachine.WriteSnapshot(JournalStreamWriter writer) + { + // Snapshots compact deduplication history, so retained history grows with the journal. + var now = TimeProvider.GetUtcNow(); + foreach (var (key, processedAt) in _processed) + { + if (MessagingTime.IsExpired(now, processedAt, Options.DeduplicationWindow)) _processed.Remove(key); + } + + Shared.InboxCodec.WriteSet(new() + { + Kind = InboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = _messages.Values.ToArray(), + Processed = _processed.Select(static pair => new InboxProcessedEntry { MessageId = pair.Key, ProcessedAt = pair.Value }).ToArray(), + DeadLetters = _deadLetters.Values.ToArray() + } + }, writer); + OnWriteCaptured(); + } + + void IDurableValueCommandHandler.ApplySet(InboxCommand command) => Apply(command); + + private void Apply(InboxCommand command) + { + switch (command.Kind) + { + case InboxCommandKind.Accept: + var message = command.Message ?? throw new InvalidOperationException("An accept command requires its message."); + _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = ++_lastSequence }); + _processed.Remove(message.Envelope.MessageId); + ReportDepth(1); + break; + case InboxCommandKind.Retry: + _messages[command.MessageId].State = command.State ?? throw new InvalidOperationException("A retry command requires its state."); + break; + case InboxCommandKind.Complete: + if (_messages.Remove(command.MessageId)) ReportDepth(-1); + _processed[command.MessageId] = command.ProcessedAt; + if (command.DeadLetter is { } letter) _deadLetters[command.MessageId] = letter; + break; + case InboxCommandKind.RemoveDeadLetters: + foreach (var key in command.RemovedKeys!) _deadLetters.Remove(key); + break; + case InboxCommandKind.Snapshot: + var snapshot = command.Snapshot ?? throw new InvalidOperationException("A snapshot command requires its state."); + var previousCount = _messages.Count; + _messages.Clear(); + _processed.Clear(); + _deadLetters.Clear(); + foreach (var restored in snapshot.Messages) _messages.Add(restored.Message.Envelope.MessageId, restored); + ReportDepth(_messages.Count - previousCount); + + foreach (var processed in snapshot.Processed) _processed.Add(processed.MessageId, processed.ProcessedAt); + foreach (var restored in snapshot.DeadLetters) _deadLetters.Add(restored.Message.Envelope.MessageId, restored); + break; + default: + throw new NotSupportedException($"Inbox command '{command.Kind}' is not supported."); + } + } +} + +internal enum InboxCommandKind +{ + Accept = 1, + Retry = 2, + Complete = 3, + RemoveDeadLetters = 4, + Snapshot = 6 +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxCommand")] +internal sealed class InboxCommand +{ + [Id(0)] public InboxCommandKind Kind { get; init; } + [Id(1)] public InboxMessage? Message { get; init; } + [Id(2)] public HierarchicalKey MessageId { get; init; } + [Id(3)] public InboxMessageState? State { get; init; } + [Id(4)] public DateTimeOffset ProcessedAt { get; init; } + [Id(5)] public InboxDeadLetter? DeadLetter { get; init; } + [Id(6)] public HierarchicalKey[]? RemovedKeys { get; init; } + [Id(7)] public InboxSnapshot? Snapshot { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxSnapshot")] +internal sealed class InboxSnapshot +{ + [Id(0)] public InboxEntry[] Messages { get; init; } = []; + [Id(1)] public InboxProcessedEntry[] Processed { get; init; } = []; + [Id(2)] public InboxDeadLetter[] DeadLetters { get; init; } = []; +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxProcessedEntry")] +internal sealed class InboxProcessedEntry +{ + [Id(0)] public HierarchicalKey MessageId { get; init; } + [Id(1)] public DateTimeOffset ProcessedAt { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxEntry")] +internal sealed class InboxEntry +{ + [Id(0)] public required InboxMessage Message { get; init; } + [Id(1)] public required InboxMessageState State { get; set; } + [field: NonSerialized] + [JsonIgnore] public long Sequence { get; init; } +} + +[JsonSerializable(typeof(InboxCommand))] +[JsonSerializable(typeof(InboxSnapshot))] +internal sealed partial class InboxJsonContext : JsonSerializerContext; diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs new file mode 100644 index 00000000000..96eaaa322b7 --- /dev/null +++ b/src/Orleans.Messaging/Inbox.cs @@ -0,0 +1,427 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Accepts, deduplicates, and processes durable incoming messages for one grain. +/// +/// +/// All state transitions run on the activation scheduler, so each check-then-emit sequence is atomic. +/// Accepted messages are processed only after their acceptance is durable. +/// +internal sealed partial class Inbox : IInbox, IInboxExtension, IStateMachine, IDurableValueCommandHandler +{ + private readonly MessagingPump _pump; + private readonly InboxDispatcher _handlers = new([]); + + public Inbox(MessagingPump pump) + { + _pump = pump; + pump.StateManager.RegisterStateMachine(MessagingStateNames.Inbox, this); + pump.Register(this); + } + + private MessagingShared Shared => _pump.Shared; + private DurableMessagingOptions Options => _pump.Shared.Options; + private ILogger Logger => _pump.Shared.InboxLogger; + private TimeProvider TimeProvider => _pump.Shared.TimeProvider; + private IGrainContext GrainContext => _pump.GrainContext; + + internal int Count => _messages.Count; + + internal IReadOnlyDictionary MessageState => _messages; + + internal IReadOnlyDictionary DeadLetterState => _deadLetters; + + internal InboxDispatcher Handlers => _handlers; + + public IInbox Register(IInboxHandler handler) + { + _handlers.Add(handler); + return this; + } + + public async ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _pump.ShutdownToken.ThrowIfCancellationRequested(); + EnvelopeValidation.Validate(message); + + // Delivery is a grain call, so the activation is servicing requests. + _pump.EnsureStarted(); + var key = message.Envelope.MessageId; + try + { + if (_processed.TryGetValue(key, out var processedAt) + && !MessagingTime.IsExpired(TimeProvider.GetUtcNow(), processedAt, Options.DeduplicationWindow)) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); + return DeliveryResult.Duplicate(); + } + + if (_messages.TryGetValue(key, out var pending)) + { + if (!EnvelopeEquivalence.AreSameCommand(pending.Message, message)) + { + throw new InvalidOperationException( + $"The inbox already contains a different command with message ID '{key}'."); + } + + // The sender may retire its copy once this returns, so the acceptance must be durable. + if (!IsDurable(pending)) await PersistAsync().ConfigureAwait(true); + _pump.Signal(); + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); + return DeliveryResult.Duplicate(); + } + + if (_messages.Count >= Options.MaxCapacity) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "backpressured"); + return DeliveryResult.Backpressured(); + } + + if (_handlers.IsEmpty) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "handler_not_found"); + LogHandlerNotFound(Logger, key, GrainContext.GrainId); + return DeliveryResult.HandlerNotFound(); + } + + Emit(new() { Kind = InboxCommandKind.Accept, Message = message }); + await PersistAsync().ConfigureAwait(true); + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); + LogMessageAccepted(Logger, key, GrainContext.GrainId); + return DeliveryResult.Accepted(); + } + catch (Exception exception) + { + LogDeliveryOperationFailed(Logger, exception, key, GrainContext.GrainId); + throw; + } + } + + /// + /// Processes one batch of durable, due messages and returns whether any message was processed. + /// + internal async ValueTask ProcessReadyAsync(CancellationToken cancellationToken) + { + if (_handlers.IsEmpty || _messages.Count == 0) + { + return false; + } + + var now = TimeProvider.GetUtcNow(); + var batch = new List(Math.Min(Options.InboxBatchSize, _messages.Count)); + foreach (var entry in _messages.Values) + { + if (IsDurable(entry) && (entry.State.NextAttemptAt is not { } at || at <= now)) + { + batch.Add(entry); + if (batch.Count == Options.InboxBatchSize) break; + } + } + + if (batch.Count == 0) + { + return false; + } + + foreach (var entry in batch) + { + cancellationToken.ThrowIfCancellationRequested(); + await ProcessAsync(entry, cancellationToken).ConfigureAwait(true); + } + + await PersistAsync().ConfigureAwait(true); + return true; + } + + private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancellationToken) + { + if (!IsCurrent(entry)) + { + return; + } + + var start = Stopwatch.GetTimestamp(); + var execution = new HandlerExecution(this, entry) { Active = true }; + Exception? error = null; + try + { + await _handlers.HandleAsync(new InboxHandlerContext(entry.Message, execution), cancellationToken).ConfigureAwait(true); + execution.ThrowIfRejected(); + if (!execution.Completed) + { + execution.Reject(new InvalidOperationException( + "Inbox handlers must call Complete or Fail before returning successfully.")); + } + } + catch (Exception) when (execution.Rejection is { } rejection) + { + _pump.Fail(rejection.SourceException, "Inbox operation failed."); + rejection.Throw(); + } + catch (Exception exception) when (execution.Completed) + { + LogHandlerException(Logger, exception, entry.Message.Envelope.MessageId, GrainContext.GrainId); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + error = exception; + LogHandlerException(Logger, exception, entry.Message.Envelope.MessageId, GrainContext.GrainId); + } + finally + { + execution.Active = false; + } + + string status; + if (error is not null) + { + if (!IsCurrent(entry)) + { + return; + } + + var attempts = checked(entry.State.AttemptCount + 1); + if (attempts >= Options.MaxProcessingAttempts) + { + StageDeadLetter(entry, error.Message, attempts); + status = "dead_lettered"; + } + else + { + Emit(new() + { + Kind = InboxCommandKind.Retry, + MessageId = entry.Message.Envelope.MessageId, + State = new() + { + AttemptCount = attempts, + LastError = error.ToString(), + NextAttemptAt = MessagingTime.AddClamped(TimeProvider.GetUtcNow(), + TimeSpan.FromTicks(Options.BackpressureRetryDelay.Ticks * (1L << Math.Min(attempts - 1, DurableMessagingOptions.MaximumBackoffExponent)))) + } + }); + status = "retry"; + } + } + else + { + status = execution.DeadLetterReason is null ? "success" : "dead_lettered"; + } + + Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, status); + Shared.Instruments.OnInboxProcessingDuration(Stopwatch.GetElapsedTime(start), Shared.GrainType); + } + + private bool IsCurrent(InboxEntry entry) => + _messages.TryGetValue(entry.Message.Envelope.MessageId, out var current) && ReferenceEquals(current, entry); + + private void StageCompletion(InboxEntry entry, InboxDeadLetter? deadLetter) => + Emit(new() + { + Kind = InboxCommandKind.Complete, + MessageId = entry.Message.Envelope.MessageId, + ProcessedAt = TimeProvider.GetUtcNow(), + DeadLetter = deadLetter + }); + + private void StageDeadLetter(InboxEntry entry, string reason, int attemptCount) + { + var now = TimeProvider.GetUtcNow(); + var key = entry.Message.Envelope.MessageId; + CompactDeadLetters(now, reservedCapacity: _deadLetters.ContainsKey(key) ? 0 : 1); + StageCompletion(entry, new InboxDeadLetter + { + Message = entry.Message, + DeadLetteredAt = now, + Reason = reason, + AttemptCount = attemptCount + }); + } + + private bool CompactDeadLetters(DateTimeOffset now, int reservedCapacity = 0) + { + if (_deadLetters.Count == 0) return false; + var retained = new Dictionary(_deadLetters); + if (!DeadLetterRetention.Compact(retained, now, Options.DeadLetterRetentionPeriod, Options.MaxRetainedDeadLetters, + static entry => entry.DeadLetteredAt, reservedCapacity)) + { + return false; + } + + Emit(new() + { + Kind = InboxCommandKind.RemoveDeadLetters, + RemovedKeys = _deadLetters.Keys.Where(key => !retained.ContainsKey(key)).ToArray() + }); + return true; + } + + internal bool RemoveDeadLetter(HierarchicalKey messageId) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + if (!_deadLetters.ContainsKey(messageId)) return false; + Emit(new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [messageId] }); + return true; + } + + /// + /// Returns the delay until the next message is due, or when the inbox is empty. + /// + internal TimeSpan? GetNextWakeupDelay() + { + if (_messages.Count == 0) return null; + var now = TimeProvider.GetUtcNow(); + var next = DateTimeOffset.MaxValue; + foreach (var entry in _messages.Values) + { + if (entry.State.NextAttemptAt is not { } at || at <= now) return TimeSpan.Zero; + if (at < next) next = at; + } + + return next - now; + } + + internal async Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (GrainContext.GrainInstance is not DurableGrain and not IMessagingGrain) + { + throw new InvalidOperationException("Inbox activation requires IMessagingGrain or DurableGrain."); + } + + if (CompactDeadLetters(TimeProvider.GetUtcNow())) + { + await PersistAsync().ConfigureAwait(true); + } + } + + /// Releases the reported depth once the activation's messaging state is discarded. + internal void ReleaseDepth() => ReportDepth(-_messages.Count); + + private void ReportDepth(int delta) => Shared.Instruments.OnInboxDepthChanged(Shared.GrainType, delta); + + private async ValueTask PersistAsync() + { + try + { + await _pump.StateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (JournaledStatePostCommitException) + { + throw; + } + catch (Exception exception) + { + _pump.Fail(exception, "Inbox persistence failed."); + throw; + } + } + + /// + /// Tracks one handler attempt and stages its terminal outcome. + /// + internal sealed class HandlerExecution(Inbox owner, InboxEntry entry) + { + public bool Active { get; set; } + + public bool Completed { get; private set; } + + public string? DeadLetterReason { get; private set; } + + public ExceptionDispatchInfo? Rejection { get; private set; } + + public void Complete() + { + ValidateAttempt(); + if (Completed) + { + return; + } + + try + { + owner.StageCompletion(entry, deadLetter: null); + Completed = true; + } + catch (Exception exception) + { + Reject(exception); + } + } + + public void Fail(string reason) + { + ValidateAttempt(); + ArgumentException.ThrowIfNullOrWhiteSpace(reason); + if (Completed) + { + if (string.Equals(DeadLetterReason, reason, StringComparison.Ordinal)) + { + return; + } + + Reject(new InvalidOperationException("The inbox handler has already staged a different terminal outcome.")); + } + + try + { + owner.StageDeadLetter(entry, reason, checked(entry.State.AttemptCount + 1)); + DeadLetterReason = reason; + Completed = true; + } + catch (Exception exception) + { + Reject(exception); + } + } + + public void ThrowIfRejected() => Rejection?.Throw(); + + [DoesNotReturn] + public void Reject(Exception exception) + { + Rejection ??= ExceptionDispatchInfo.Capture(exception); + Rejection.Throw(); + } + + private void ValidateAttempt() + { + ThrowIfRejected(); + if (!Active) + { + Reject(new InvalidOperationException("The inbox handler context belongs to an inactive or different attempt.")); + } + } + } + + [LoggerMessage(Level = LogLevel.Error, EventName = "DeliveryOperationFailed", + Message = "Inbox delivery of message {MessageId} to {GrainId} failed")] + private static partial void LogDeliveryOperationFailed(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId grainId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "No inbox handler registered for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Information, Message = "Accepted message {MessageId} to {ReceiverId}")] + private static partial void LogMessageAccepted(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] + private static partial void LogHandlerException(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); +} diff --git a/src/Orleans.Messaging/InboxHandlerContext.cs b/src/Orleans.Messaging/InboxHandlerContext.cs new file mode 100644 index 00000000000..ed8b74b2857 --- /dev/null +++ b/src/Orleans.Messaging/InboxHandlerContext.cs @@ -0,0 +1,10 @@ +namespace Orleans.Messaging; + +internal sealed class InboxHandlerContext(InboxMessage message, Inbox.HandlerExecution execution) : IInboxHandlerContext +{ + public InboxMessage Message { get; } = message; + + public void Complete() => execution.Complete(); + + public void Fail(string reason) => execution.Fail(reason); +} diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs new file mode 100644 index 00000000000..0fa6e5401ad --- /dev/null +++ b/src/Orleans.Messaging/MessageState.cs @@ -0,0 +1,33 @@ +using System; + +namespace Orleans.Messaging; + +[GenerateSerializer, Alias("Orleans.Messaging.InboxMessageState")] +internal sealed class InboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + +} + +[GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] +internal sealed class InboxDeadLetter +{ + [Id(0)] + public required InboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} diff --git a/src/Orleans.Messaging/MessagingActivationValidator.cs b/src/Orleans.Messaging/MessagingActivationValidator.cs new file mode 100644 index 00000000000..5a8279b4d9a --- /dev/null +++ b/src/Orleans.Messaging/MessagingActivationValidator.cs @@ -0,0 +1,20 @@ +using System; +using Orleans.Concurrency; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal static class MessagingActivationValidator +{ + // The public attribute exposes the runtime's internal, sealed strategy type. + private static readonly Type _statelessWorkerPlacementType = new StatelessWorkerAttribute().PlacementStrategy.GetType(); + + public static void Validate(Type grainClass, PlacementStrategy placementStrategy) + { + if (placementStrategy.GetType() == _statelessWorkerPlacementType) + { + throw new InvalidOperationException( + $"Messaging requires one activation per grain identity, but grain type '{grainClass}' is a stateless worker."); + } + } +} diff --git a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs new file mode 100644 index 00000000000..81d6460fcdf --- /dev/null +++ b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs @@ -0,0 +1,31 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingGrainTypeConfigurator(GrainClassMap grainClasses, IServiceProvider services) : IConfigureGrainTypeComponents +{ + public void Configure(GrainType grainType, GrainProperties properties, GrainTypeSharedContext shared) + { + if (!grainClasses.TryGetGrainClass(grainType, out var grainClass)) + { + throw new InvalidOperationException($"No grain implementation is registered for '{grainType}'."); + } + + if (typeof(DurableGrain).IsAssignableFrom(grainClass) || typeof(IMessagingGrain).IsAssignableFrom(grainClass)) + { + var placementStrategy = shared.PlacementStrategy; + shared.SetComponent(new MessagingShared(services, grainType)); + shared.AddActivationSetup(context => + { + MessagingActivationValidator.Validate(grainClass, placementStrategy); + var activationServices = context.ActivationServices; + var inbox = activationServices.GetRequiredService(); + context.GetComponent()!.GetOrSetExtension(() => inbox); + }); + } + } +} diff --git a/src/Orleans.Messaging/MessagingInstruments.cs b/src/Orleans.Messaging/MessagingInstruments.cs new file mode 100644 index 00000000000..852a23a68ec --- /dev/null +++ b/src/Orleans.Messaging/MessagingInstruments.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +internal sealed class MessagingInstruments(OrleansInstruments instruments) +{ + private const string MillisecondsUnit = "ms"; + private const string GrainTypeTagName = "grain_type"; + private const string StatusTagName = "status"; + + private readonly Counter _inboxMessagesReceived = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-received"); + private readonly Counter _inboxMessagesProcessed = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-processed"); + private readonly Histogram _inboxProcessingDuration = instruments.Meter.CreateHistogram("orleans-messaging-inbox-processing-duration", MillisecondsUnit); + private readonly UpDownCounter _inboxDepth = instruments.Meter.CreateUpDownCounter("orleans-messaging-inbox-depth"); + + internal static MessagingInstruments CreateForDirectConstruction() => new(new OrleansInstruments(new DirectMeterFactory())); + + internal void OnInboxDepthChanged(string grainType, int delta) => AddDepth(_inboxDepth, grainType, delta); + + private static void AddDepth(UpDownCounter counter, string grainType, int delta) + { + if (delta != 0 && counter.Enabled) + { + counter.Add(delta, new KeyValuePair(GrainTypeTagName, grainType)); + } + } + + internal void OnInboxMessageReceived(string grainType, string status) => + Add(_inboxMessagesReceived, grainType, status); + + internal void OnInboxMessageProcessed(string grainType, string status) => + Add(_inboxMessagesProcessed, grainType, status); + + internal void OnInboxProcessingDuration(TimeSpan duration, string grainType) => + Record(_inboxProcessingDuration, duration, grainType); + + private static void Add(Counter counter, string grainType, string status) + { + if (counter.Enabled) + { + counter.Add(1, new(GrainTypeTagName, grainType), new(StatusTagName, status)); + } + } + + private static void Record(Histogram histogram, TimeSpan duration, string grainType) + { + if (histogram.Enabled) + { + histogram.Record(Math.Max(0, duration.TotalMilliseconds), new KeyValuePair(GrainTypeTagName, grainType)); + } + } + + private sealed class DirectMeterFactory : IMeterFactory + { + public Meter Create(MeterOptions options) => new(options); + + public void Dispose() + { + } + } +} diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs new file mode 100644 index 00000000000..8489a15ab64 --- /dev/null +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -0,0 +1,231 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Drives inbox processing for one activation. +/// +/// +/// A single loop runs on the activation scheduler and services the inbox whenever it is signaled. +/// One durable job keeps waking the grain while the inbox holds work. Wake-ups may be early, +/// late, or duplicated: every pass derives the work to perform from current state. +/// +internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecycleObserver +{ + internal const string JobName = "orleans.messaging.pump"; + private static readonly TimeSpan RecheckDelay = TimeSpan.FromSeconds(1); + private readonly SingleWaiterAutoResetEvent _signal = new() { RunContinuationsAsynchronously = true }; + private readonly CancellationTokenSource _shutdown = new(); + private Inbox? _inbox; + private Task? _loop; + private TaskCompletionSource? _nextPass; + private Task? _scheduling; + private bool _wakeupScheduled; + + public MessagingPump(IGrainContext grainContext, IJournaledStateManager stateManager, IDurableJobHandlerRegistry jobHandlers) + { + GrainContext = grainContext; + StateManager = stateManager; + Shared = grainContext.GetComponent() + ?? throw new InvalidOperationException($"Messaging is not configured for grain type '{grainContext.GrainId.Type}'."); + jobHandlers.Register(this); + grainContext.ObservableLifecycle.Subscribe(nameof(MessagingPump), GrainLifecycleStage.Activate, this); + + // Recovery reports depth during SetupState, so releasing it there covers activations which fail before Activate. + grainContext.ObservableLifecycle.Subscribe(nameof(MessagingPump), GrainLifecycleStage.SetupState, static _ => Task.CompletedTask, _ => + { + _inbox?.ReleaseDepth(); + return Task.CompletedTask; + }); + } + + public IGrainContext GrainContext { get; } + + public IJournaledStateManager StateManager { get; } + + public MessagingShared Shared { get; } + + /// Gets the token which is canceled when messaging stops for this activation. + public CancellationToken ShutdownToken => _shutdown.Token; + + internal void Register(Inbox inbox) => _inbox = inbox; + + private Inbox Inbox => _inbox ?? throw new InvalidOperationException("The messaging inbox has not been registered."); + + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); + + /// Requests a processing pass. + public void Signal() + { + if (!_shutdown.IsCancellationRequested) _signal.Signal(); + } + + /// Starts the loop once the activation is servicing grain calls. + public void EnsureStarted() + { + if (_loop is null && !_shutdown.IsCancellationRequested) + { + _loop = RunAsync(); + } + } + + /// Ensures a durable job will wake this grain while messaging work remains. + public async ValueTask EnsureWakeupAsync(CancellationToken cancellationToken) + { + if (_wakeupScheduled) return; + var scheduling = _scheduling ??= Shared.JobManager.ScheduleJobAsync(new ScheduleJobRequest + { + Target = GrainContext.GrainId, + JobName = JobName, + DueTime = Shared.JobTimeProvider.GetUtcNow() + }, _shutdown.Token); + try + { + await scheduling.WaitAsync(cancellationToken).ConfigureAwait(true); + _wakeupScheduled = true; + } + finally + { + if (scheduling.IsCompleted && ReferenceEquals(_scheduling, scheduling)) _scheduling = null; + } + } + + public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) + { + // A running job keeps rescheduling itself until the inbox is empty. + _wakeupScheduled = true; + EnsureStarted(); + var pass = _nextPass ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + _signal.Signal(); + try + { + await pass.Task.WaitAsync(cancellationToken).ConfigureAwait(true); + } + catch + { + // This attempt no longer guarantees a future wake-up. + _wakeupScheduled = false; + if (!_shutdown.IsCancellationRequested && Inbox.Count > 0) + { + EnsureWakeupAsync(CancellationToken.None).AsTask().Ignore(); + } + + throw; + } + + if (Inbox.GetNextWakeupDelay() is not { } delay) + { + _wakeupScheduled = false; + return DurableJobRunResult.Completed; + } + + return DurableJobRunResult.RescheduleAt( + MessagingTime.AddClamped(Shared.JobTimeProvider.GetUtcNow(), delay > TimeSpan.Zero ? delay : RecheckDelay)); + } + + private async Task RunAsync() + { + var cancellationToken = _shutdown.Token; + while (!cancellationToken.IsCancellationRequested) + { + var pass = _nextPass; + _nextPass = null; + var progress = false; + try + { + progress = await Inbox.ProcessReadyAsync(cancellationToken).ConfigureAwait(true); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + progress = false; + LogPassFailed(Shared.PumpLogger, exception, GrainContext.GrainId); + } + + pass?.TrySetResult(); + if (!progress) + { + await _signal.WaitAsync().ConfigureAwait(true); + } + } + + _nextPass?.TrySetCanceled(cancellationToken); + } + + public async Task OnStart(CancellationToken cancellationToken) + { + await Inbox.StartAsync(cancellationToken).ConfigureAwait(true); + if (Inbox.Count > 0) + { + await EnsureWakeupAsync(cancellationToken).ConfigureAwait(true); + } + } + + public async Task OnStop(CancellationToken cancellationToken) + { + var scheduling = _scheduling; + Stop(); + + // Grain calls finish before deactivation stops the lifecycle, so the loop owns every remaining write. + if (_loop is { } loop) + { + await loop.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + + if (scheduling is not null) + { + await ((Task)scheduling).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + } + + /// Stops processing for this activation. + internal void Stop() + { + if (_shutdown.IsCancellationRequested) return; + try + { + _shutdown.Cancel(); + } + catch (AggregateException exception) + { + LogCancellationFailed(Shared.PumpLogger, exception); + } + finally + { + _signal.Signal(); + } + } + + /// Stops messaging and deactivates the grain after an unrecoverable failure. + internal void Fail(Exception exception, string description) + { + if (_shutdown.IsCancellationRequested) return; + Stop(); + try + { + GrainContext.Deactivate(new DeactivationReason(DeactivationReasonCode.ApplicationError, exception, description)); + } + catch (Exception deactivationException) + { + LogDeactivationFailed(Shared.PumpLogger, deactivationException); + } + } + + + [LoggerMessage(Level = LogLevel.Error, Message = "Messaging processing pass failed for grain {GrainId}")] + private static partial void LogPassFailed(ILogger logger, Exception exception, GrainId grainId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Requesting deactivation after a messaging failure failed.")] + private static partial void LogDeactivationFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "A messaging cancellation callback failed during shutdown.")] + private static partial void LogCancellationFailed(ILogger logger, Exception exception); +} diff --git a/src/Orleans.Messaging/MessagingShared.cs b/src/Orleans.Messaging/MessagingShared.cs new file mode 100644 index 00000000000..c76340e45eb --- /dev/null +++ b/src/Orleans.Messaging/MessagingShared.cs @@ -0,0 +1,55 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Services and settings shared by every messaging activation of one grain type. +/// +/// +/// stores one instance as a grain-type component, so activations +/// resolve it as an component instead of copying it per activation. +/// +internal sealed class MessagingShared +{ + public MessagingShared(IServiceProvider services, GrainType grainType) + { + GrainType = grainType.ToString(); + Options = services.GetRequiredService>().Value; + TimeProvider = services.GetRequiredService(); + JobTimeProvider = services.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs); + JobManager = services.GetRequiredService(); + Instruments = services.GetRequiredService(); + var loggers = services.GetRequiredService(); + InboxLogger = loggers.CreateLogger(); + PumpLogger = loggers.CreateLogger(); + var format = services.GetRequiredService>().Value.JournalFormatKey; + InboxCodec = services.GetRequiredKeyedService>(format); + } + + /// Gets the grain type metric tag. + public string GrainType { get; } + + public DurableMessagingOptions Options { get; } + + public TimeProvider TimeProvider { get; } + + /// Gets the durable jobs clock, which defines the time domain of wake-up due times. + public TimeProvider JobTimeProvider { get; } + + public ILocalDurableJobManager JobManager { get; } + + public MessagingInstruments Instruments { get; } + + public ILogger InboxLogger { get; } + + public ILogger PumpLogger { get; } + + public IDurableValueCommandCodec InboxCodec { get; } +} diff --git a/src/Orleans.Messaging/MessagingStateNames.cs b/src/Orleans.Messaging/MessagingStateNames.cs new file mode 100644 index 00000000000..1b947f730d9 --- /dev/null +++ b/src/Orleans.Messaging/MessagingStateNames.cs @@ -0,0 +1,8 @@ +namespace Orleans.Messaging; + +internal static class MessagingStateNames +{ + private const string Prefix = "orleans.messaging."; + + public const string Inbox = Prefix + "inbox"; +} diff --git a/src/Orleans.Messaging/MessagingTime.cs b/src/Orleans.Messaging/MessagingTime.cs new file mode 100644 index 00000000000..c185c232926 --- /dev/null +++ b/src/Orleans.Messaging/MessagingTime.cs @@ -0,0 +1,21 @@ +using System; + +namespace Orleans.Messaging; + +internal static class MessagingTime +{ + public static bool IsExpired(DateTimeOffset now, DateTimeOffset timestamp, TimeSpan retention) + { + var nowTicks = now.UtcTicks; + var timestampTicks = timestamp.UtcTicks; + return nowTicks >= timestampTicks + && nowTicks - timestampTicks >= retention.Ticks; + } + + public static DateTimeOffset AddClamped(DateTimeOffset timestamp, TimeSpan duration) + { + var utcTicks = timestamp.UtcDateTime.Ticks; + var remainingTicks = DateTimeOffset.MaxValue.Ticks - utcTicks; + return new DateTimeOffset(utcTicks + Math.Min(duration.Ticks, remainingTicks), TimeSpan.Zero); + } +} diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj index 801ea472629..0ab2b668c47 100644 --- a/src/Orleans.Messaging/Orleans.Messaging.csproj +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -24,9 +24,12 @@ + + + diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 0e06852d74a..d4bd00ba98c 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -1,9 +1,23 @@ # Microsoft Orleans Messaging -`Microsoft.Orleans.Messaging` defines the contracts for grain-scoped durable -inboxes and outboxes in Microsoft Orleans: envelopes, hierarchical message -identities, inbox and outbox interfaces, handler, sink, and source registration, -typed message bindings, delivery results, and options. +`Microsoft.Orleans.Messaging` provides durable, grain-scoped inbox processing for +Microsoft Orleans, built on Orleans Journaling and Durable Jobs, together with the +messaging contracts for inboxes and outboxes. A grain's inbox accepts messages durably, +persists them with the grain's other journaled state, processes each at least once, +and deduplicates them by message ID. + +## Setup + +Configure Journaling and Durable Jobs storage, then call `AddMessaging` on the silo +builder or service collection. The optional `DurableMessagingOptions` callback configures +capacity, batch sizes, retries, deduplication, and dead-letter retention; options are +validated at startup. Register typed subjects with `AddMessageType(subject)`. + +Messaging grains derive from `DurableGrain` or implement `IMessagingGrain`, and +constructor-inject `IInbox`. During activation, the messaging grain-type +configurator installs the grain's inbox extension. Stateless worker grains are +rejected; reentrant and interleaving grains are supported, and the grain's code is +responsible for correctness when calls interleave. ## Messages @@ -45,8 +59,7 @@ inbox.Register(reserve, this, static (request, grain, context) => grain.HandleRe ``` Overloads accept synchronous or asynchronous delegates, with or without a `TArg` -argument for static, allocation-free delegates. Register typed subjects with -`AddMessageType(subject)`. +argument for static, allocation-free delegates. A handler finishes each message with `IInboxHandlerContext.Complete()` or `Fail(reason)`: @@ -54,6 +67,13 @@ A handler finishes each message with `IInboxHandlerContext.Complete()` or - `Complete()` stages the message's removal and completion record. - `Fail(reason)` moves the message to the inbox dead letters and records its ID as processed. +- A handler exception before either call retries the message with exponential + backoff, and dead-letters it after `MaxProcessingAttempts`. +- Returning without either call, or calling both, deactivates the grain. + +The inbox persists each batch's outcomes and business changes in one journal write. +Perform asynchronous and fallible preparation with local values, then apply state +changes and complete in one synchronous block. ## Send messages @@ -94,9 +114,11 @@ construction or activation setup, before journal recovery. A sink that takes a scheduled message honors its `DeliverAt`. Sinks are idempotent. -## Delivery results +## Delivery and processing -Each receiver returns a `DeliveryResult`: +`IInboxExtension.DeliverAsync` delivers a message to a grain's inbox. Accepted inbox +messages are processed only after the receiver's journal write completes. Each +receiver returns a `DeliveryResult`: - `Accepted`: the message is durable in the inbox, or, for a message an inbox sink accepted, in the sink's state together with its processed record. @@ -104,7 +126,27 @@ Each receiver returns a `DeliveryResult`: - `Backpressured`: the inbox holds `MaxCapacity` pending messages without `DeliverAt`. The inbox admits scheduled messages regardless of `MaxCapacity`. - `HandlerNotFound`: the receiver has no registered handlers. -- `DeadLettered`: the message moved to the receiver's dead letters. -`DurableMessagingOptions` configures capacity, batch sizes, retries, deduplication, -and dead-letter retention. +Each activation runs one messaging pump on its scheduler, processing inbox batches. +One durable job per grain wakes the pump while the inbox holds work. Journaled state determines which messages are due, so each +wake-up processes exactly the ready work. + +## State + +The inbox is an event-sourced journal state named `orleans.messaging.inbox`, and works +with the JSON and Orleans binary journal formats. It persists alongside the grain's +other journaled state. After the grain's state is deleted, it starts empty and +continues working. + +Completed message IDs are retained for `DeduplicationWindow`. Dead letters are +retained up to `DeadLetterRetentionPeriod` and `MaxRetainedDeadLetters`. + +## Diagnostics + +`IMessagingDiagnostics` exposes `InboxDeadLetters`, with each record's message, +reason, attempt count, and dead-letter time. `RemoveInboxDeadLetter` removes a record, +durably at the grain's next journal write. + +The `Microsoft.Orleans` meter publishes counters for received and processed messages +tagged by grain type and status, a histogram for processing duration, and the +`orleans-messaging-inbox-depth` up-down counter for pending work. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 426e51b416b..37a59fbf816 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -12,10 +12,28 @@ public static partial class MessageTypeExtensions { public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddMessageType(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, string subject) { throw null; } } + + public static partial class MessagingExtensions + { + public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddMessaging(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, System.Action? configureOptions = null) { throw null; } + + public static ISiloBuilder AddMessaging(this ISiloBuilder builder, System.Action? configureOptions = null) { throw null; } + } } namespace Orleans.Messaging { + public sealed partial class DeadLetter + { + public int AttemptCount { get { throw null; } init { } } + + public System.DateTimeOffset DeadLetteredAt { get { throw null; } init { } } + + public required TMessage Message { get { throw null; } init { } } + + public required string Reason { get { throw null; } init { } } + } + [GenerateSerializer] [Alias("Orleans.Messaging.DeliveryResult")] public readonly partial struct DeliveryResult @@ -179,6 +197,17 @@ public partial interface IMessagingExtension : Runtime.IGrainExtension, Runtime. System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); } + public partial interface IMessagingDiagnostics + { + System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } + + bool RemoveInboxDeadLetter(HierarchicalKey messageId); + } + + public partial interface IMessagingGrain + { + } + public static partial class InboxExtensions { public static IInbox Register(this IInbox inbox, MessageType messageType, System.Action handler) { throw null; } diff --git a/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs new file mode 100644 index 00000000000..3cf35285602 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/ActivationValidationTests.cs @@ -0,0 +1,99 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Orleans.Placement; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ActivationValidationTests +{ + [Fact] + public void StatelessWorker_FailsWithSpecificDiagnostic() + { + var exception = Assert.Throws(() => + MessagingActivationValidator.Validate(typeof(StatelessGrain), new StatelessWorkerAttribute().PlacementStrategy)); + + Assert.Contains("one activation", exception.Message, StringComparison.Ordinal); + Assert.Contains(typeof(StatelessGrain).ToString(), exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(typeof(ReentrantGrain))] + [InlineData(typeof(MayInterleaveGrain))] + [InlineData(typeof(InterleavableGrain))] + public void ReentrantAndInterleavingModels_AreAccepted(Type grainClass) => + MessagingActivationValidator.Validate(grainClass, new RandomPlacementAttribute().PlacementStrategy); + + [Fact] + public async Task StandardStateConstruction_MissingSelectedCodecFailsBeforeEnrollment() + { + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.RemoveAllKeyed(typeof(IDurableValueCommandCodec<>), selectedFormat); + var id = new JournalId("missing-codec/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + Assert.Contains(selectedFormat, exception.Message, StringComparison.Ordinal); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Fact] + public async Task CodecResolution_PreservesUnrelatedFailure() + { + var cause = new InvalidOperationException("codec resolution failure"); + const string selectedFormat = "orleans-binary"; + var builder = InboxStateManagerBoundaryTests.CreateBuilder(selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (_, _) => throw cause); + var id = new JournalId("codec-resolution/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + await using var provider = builder.Services.BuildServiceProvider(validateScopes: true); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + + var exception = Assert.Throws(() => + scope.ServiceProvider.GetRequiredKeyedService>("state")); + + Assert.Same(cause, exception); + Assert.False(owner.TryGetStateMachine("state", out _)); + } + + [Reentrant] + private sealed class ReentrantGrain; + + [StatelessWorker] + private sealed class StatelessGrain; + + [MayInterleave(nameof(Interleave))] + private sealed class MayInterleaveGrain + { + public static bool Interleave(IInvokable request) => true; + } + + public interface IInterleavableBase + { + [AlwaysInterleave] + Task PingAsync(); + } + + public interface IInterleavableGrain : IGrain, IInterleavableBase; + + private sealed class InterleavableGrain : IInterleavableGrain + { + public Task PingAsync() => Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index e8b2057fe53..48d6c014d12 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -9,6 +9,37 @@ namespace Orleans.Messaging.Tests.Contracts; [TestArea("Messaging")] public sealed class DeliveryAndOptionsContractTests { + [Fact] + public void RetentionTimeArithmetic_HandlesMaximumDurationsWithoutOverflow() + { + var timestamp = DateTimeOffset.MaxValue - TimeSpan.FromTicks(1); + var fullDateTimeRange = TimeSpan.FromTicks(DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + + Assert.False(MessagingTime.IsExpired(DateTimeOffset.MaxValue, timestamp, TimeSpan.MaxValue)); + Assert.True(MessagingTime.IsExpired(DateTimeOffset.MaxValue, DateTimeOffset.MinValue, fullDateTimeRange)); + Assert.False(MessagingTime.IsExpired(DateTimeOffset.MaxValue, DateTimeOffset.MinValue, TimeSpan.MaxValue)); + Assert.False(MessagingTime.IsExpired(DateTimeOffset.MinValue, DateTimeOffset.MaxValue, TimeSpan.FromTicks(1))); + Assert.Equal(DateTimeOffset.MaxValue, MessagingTime.AddClamped(timestamp, TimeSpan.MaxValue)); + } + + [Fact] + public void DeadLetterCompaction_SaturatesMaximumRetentionWithoutOverflow() + { + var entries = new Dictionary + { + ["oldest"] = DateTimeOffset.MinValue, + ["newest"] = DateTimeOffset.MaxValue + }; + + Assert.False(DeadLetterRetention.Compact(entries, DateTimeOffset.MaxValue, TimeSpan.MaxValue, int.MaxValue, static timestamp => timestamp)); + Assert.Equal(2, entries.Count); + + var fullDateTimeRange = TimeSpan.FromTicks(DateTimeOffset.MaxValue.UtcTicks - DateTimeOffset.MinValue.UtcTicks); + Assert.True(DeadLetterRetention.Compact(entries, DateTimeOffset.MaxValue, fullDateTimeRange, int.MaxValue, static timestamp => timestamp)); + Assert.DoesNotContain("oldest", entries); + Assert.Contains("newest", entries); + } + [Fact] public void DeliveryResult_EachFactory_PreservesStatusAndPayload() { diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 29c81175fff..237b69bd9ee 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -167,6 +167,7 @@ public void PayloadUsesOrdinaryExternallyBoundOrleansSerializer() [Theory] [InlineData("InboxMessage")] [InlineData("OutboxMessage")] + [InlineData("InboxDeadLetter")] public void MessageAndDeadLetterCopyPreservesMetadataAndIsolation(string kind) { using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs new file mode 100644 index 00000000000..0e15996db72 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -0,0 +1,65 @@ +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingCommandSerializationTests +{ + private static readonly DateTimeOffset Now = new(2040, 1, 2, 3, 4, 5, TimeSpan.Zero); + + public static TheoryData InboxCommands => new(Enumerable.Range(0, CreateInboxCommands().Length)); + + [Theory] + [MemberData(nameof(InboxCommands))] + public void InboxCommand_RoundTripsThroughJsonAndOrleansSerialization(int index) + { + var command = CreateInboxCommands()[index]; + var json = JsonSerializer.Serialize(command, InboxJsonContext.Default.InboxCommand); + var fromJson = JsonSerializer.Deserialize(json, InboxJsonContext.Default.InboxCommand)!; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(command)); + + Assert.Equal(command.Kind, fromJson.Kind); + Assert.Equal(json, JsonSerializer.Serialize(fromJson, InboxJsonContext.Default.InboxCommand)); + Assert.Equal(json, JsonSerializer.Serialize(fromBinary!, InboxJsonContext.Default.InboxCommand)); + } + + private static InboxMessage InboxMessage(string id) => new() { Envelope = Envelope(id) }; + + private static Envelope Envelope(string id) => new() + { + MessageId = HierarchicalKey.Create("orders", id), + Payload = [1, 2, 3], + Subject = "orders.v1", + Headers = new(StringComparer.Ordinal) { ["trace"] = [4, 5] } + }; + + private static InboxCommand[] CreateInboxCommands() + { + var deadLetter = new InboxDeadLetter { Message = InboxMessage("dead"), DeadLetteredAt = Now, Reason = "failed", AttemptCount = 3 }; + return + [ + new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("accept"), State = new() }, + new() { Kind = InboxCommandKind.Retry, MessageId = HierarchicalKey.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom" } }, + new() { Kind = InboxCommandKind.Complete, MessageId = HierarchicalKey.Create("orders", "done"), ProcessedAt = Now, DeadLetter = deadLetter }, + new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [HierarchicalKey.Create("orders", "dead")] }, + new() + { + Kind = InboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = [new() { Message = InboxMessage("pending"), State = new() { AttemptCount = 1 } }], + Processed = [new() { MessageId = HierarchicalKey.Create("orders", "done"), ProcessedAt = Now }], + DeadLetters = [deadLetter] + } + } + ]; + } +} diff --git a/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs new file mode 100644 index 00000000000..6af1bae3c74 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Contracts/StandardJournaledStateTests.cs @@ -0,0 +1,105 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Functional; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Contracts; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class StandardJournaledStateTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Dictionary_WriteAcknowledgementPreservesLaterMutations(bool snapshot) + { + var id = new JournalId("standard-dictionary/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var items = scope.ServiceProvider.GetRequiredKeyedService>("state"); + Assert.False(owner is IDurableStateManager); + Assert.True(owner.TryGetStateMachine("state", out var registered)); + Assert.Same(items, registered); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + items.Add("first", 1); + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + items["later"] = 2; + Assert.False(first.IsCompleted); + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new KeyValuePair("first", 1), Assert.Single(persisted)); + } + Assert.Equal(2, items["later"]); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(new[] { new KeyValuePair("first", 1), new KeyValuePair("later", 2) }, final.OrderBy(pair => pair.Key)); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Value_WriteAcknowledgementPreservesNewerPendingValue(bool snapshot) + { + var id = new JournalId("standard-value/" + Guid.NewGuid().ToString("N")); + var storage = new ControlledJournalStorageProvider(); + await using var provider = CreateProvider(storage, id); + await using var scope = provider.CreateAsyncScope(); + var owner = scope.ServiceProvider.GetRequiredService(); + var value = scope.ServiceProvider.GetRequiredKeyedService>("state"); + await owner.InitializeAsync(TestContext.Current.CancellationToken); + value.Value = 1; + if (snapshot) storage.RequestSnapshot(id); + using var blocked = storage.BlockWrite(id); + var first = owner.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await blocked.WaitUntilEnteredAsync(); + value.Value = 2; + blocked.Release(); + await first; + await using (var replay = provider.CreateAsyncScope()) + { + var recovered = replay.ServiceProvider.GetRequiredService(); + var persisted = replay.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, persisted.Value); + } + Assert.Equal(2, value.Value); + await owner.WriteStateAsync(TestContext.Current.CancellationToken); + await using var finalScope = provider.CreateAsyncScope(); + var finalOwner = finalScope.ServiceProvider.GetRequiredService(); + var final = finalScope.ServiceProvider.GetRequiredKeyedService>("state"); + await finalOwner.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, final.Value); + Assert.Equal(2, storage.GetSuccessfulWriteCount(id)); + } + + private static ServiceProvider CreateProvider(ControlledJournalStorageProvider storage, JournalId id) + { + var builder = InboxStateManagerBoundaryTests.CreateBuilder("orleans-binary"); + builder.Services.AddSingleton(sp => + { + storage.Configure(sp.GetRequiredService>()); + return storage; + }); + builder.Services.AddScoped(sp => + sp.GetRequiredService().CreateStandalone(id)); + return builder.Services.BuildServiceProvider(validateScopes: true); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs new file mode 100644 index 00000000000..a1a5b3fc994 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs @@ -0,0 +1,109 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ByteMessagingPayloadTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task RpcAcceptanceAndDuplicate_CopyPayloadAndPreserveCallerBytesAcrossReplay() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(302, "rpc-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + using var handler = new PayloadProbe(expected); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.NotSame(envelope.Envelope.Payload, handler.Payload); + envelope.Envelope.Payload[0] ^= 0xff; + Assert.Equal(expected, handler.Payload.ToArray()); + envelope.Envelope.Payload[0] ^= 0xff; + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + await handler.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.InboxCount == 0 && snapshot.ProcessedMessageCount == 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task RpcRejection_PreservesCallerBytesWithoutStagingState() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + var original = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(301, "rejected-bytes")); + var expected = envelope.Envelope.Payload.ToArray(); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(receiver, envelope)).Status); + var rejected = await receiver.GetSnapshotAsync(); + Assert.Equal(0, rejected.InboxCount); + Assert.Equal(0, rejected.ProcessedMessageCount); + Assert.Empty(rejected.Effects); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(expected, envelope.Envelope.Payload.ToArray()); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completed.SetResult(); } + catch (Exception exception) { completed.SetException(exception); } + }); + return completed.Task; + } + + private sealed class PayloadProbe(byte[] expected) : IInboxHandler, IDisposable + { + public byte[] Payload { get; private set; } = null!; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool CanHandle(IInboxHandlerContext context) => true; + + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + try + { + Payload = context.Message.Envelope.Payload; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + Assert.Equal(expected, Payload.ToArray()); + context.Complete(); + Assert.Same(Payload, context.Message.Envelope.Payload); + Assert.Equal(expected, context.Message.Envelope.Payload.ToArray()); + Completed.TrySetResult(); + } + catch (Exception exception) + { + Completed.TrySetException(exception); + throw; + } + } + + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs new file mode 100644 index 00000000000..5f0336e18f3 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs @@ -0,0 +1,107 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DedupeExpiryClusterCollection : ICollectionFixture +{ + public const string Name = "Messaging dedupe expiry cluster"; +} + +[Collection(DedupeExpiryClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class DedupeExpiryBehaviorTests(DedupeExpiryClusterFixture fixture) +{ + [Fact] + public async Task IdleReplay_AtDeduplicationBoundary_IsAcceptedWithoutCompactionTrigger() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var original = new TestMessage(TestApplicationProtocol.NewMessageId(), 15, "expires"); + var first = CreateEnvelope(receiver, original); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, first)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + + fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + var state = await fixture.WaitForEffectCountAsync(receiver, 2); + + Assert.Equal(2, state.Effects.Single(effect => effect.LogicalId == original.LogicalId).Count); + } + + [Fact] + public async Task ExpiryReplacementFailure_ReplaysDedupeRecordBeforeExpiryMaintenance() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 16, "failed-expiry-replacement")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await fixture.WaitForEffectCountAsync(receiver, 1); + await WaitForIdleInboxAsync(receiver); + fixture.Clock.Advance(TimeSpan.FromMinutes(10)); + var oldContext = fixture.GetGrainContext(receiver); + fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await Assert.ThrowsAnyAsync(() => DeliverAsync(receiver, envelope)); + + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var failed = await receiver.GetSnapshotAsync(); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, Assert.Single(failed.Effects).Count); + var replayed = Assert.IsType(fixture.GetGrainContext(receiver).GrainInstance).ReplayedSnapshot; + Assert.Equal(1, Assert.IsType(replayed).ProcessedMessageCount); + // Expired dedupe records remain until a snapshot compacts them, but they no longer deduplicate. + Assert.Equal(1, failed.ProcessedMessageCount); + + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var retried = await fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(2, Assert.Single(retried.Effects).Count); + } + + private Task WaitForIdleInboxAsync(IMessagingTestGrain receiver) => + fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0); + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); + + private InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + HierarchicalKey? messageId = null) + { + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("expiry-test-sender", "stable"); + var built = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/expiry", message); + if (messageId is { } id) + { + built = TestApplicationProtocol.With(built, sessions, messageId: id); + } + + return built; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs new file mode 100644 index 00000000000..59eb2994701 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/EarlyActivationValidationTests.cs @@ -0,0 +1,82 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class EarlyActivationValidationTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task StatelessWorker_FailsBeforeJournalInitializationReplayAndWork() + { + var grainType = typeof(StatelessActivationValidationTestGrain); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + await AssertRejectedBeforeRecoveryAsync(grain, grainType, "one activation"); + } + + [Theory] + [InlineData(typeof(ReentrantActivationValidationTestGrain))] + [InlineData(typeof(MayInterleaveActivationValidationTestGrain))] + [InlineData(typeof(AlwaysInterleaveActivationValidationTestGrain))] + [InlineData(typeof(MetadataReentrantActivationValidationGrain))] + [InlineData(typeof(MetadataMayInterleaveActivationValidationGrain))] + public async Task ReentrantAndInterleavingModels_InitializeAndReplayNormally(Type grainType) + { + await AssertSupportedReplayAsync(grainType); + } + + private async Task AssertRejectedBeforeRecoveryAsync(IActivationValidationTestGrain grain, Type grainType, string diagnostic) + { + var failure = await Assert.ThrowsAnyAsync(() => grain.IncrementAsync()); + Assert.Contains(diagnostic, failure.ToString(), StringComparison.Ordinal); + Assert.Contains(grainType.Name, failure.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(observation.InstanceAvailableDuringConstruction); + Assert.Equal(0, observation.Activated); + Assert.Equal(0, observation.Calls); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grain.GetGrainId())); + } + + [Fact] + public async Task SupportedModel_InitializesAndReplaysFreshActivationNormally() + { + await AssertSupportedReplayAsync(typeof(SupportedActivationValidationTestGrain)); + } + + [Fact] + public async Task ResolvedNonReentrantMetadata_InitializesAndReplaysNormally() + { + await AssertSupportedReplayAsync(typeof(MetadataNonReentrantActivationValidationGrain)); + } + + private async Task AssertSupportedReplayAsync(Type grainType) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainType.FullName!); + Assert.Equal(1, await grain.IncrementAsync()); + var first = Assert.Single(Fixture.ActivationProbe.Get(grain.GetGrainId())); + Assert.False(first.InstanceAvailableDuringConstruction); + Assert.NotNull(first.Context.GrainInstance); + Assert.Equal(1, first.Activated); + Assert.Equal(1, first.Calls); + Assert.Equal(1, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(2, await grain.IncrementAsync()); + var observations = Fixture.ActivationProbe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + Assert.NotSame(first.Context, observations[1].Context); + Assert.Equal(1, observations[1].Activated); + Assert.Equal(1, observations[1].Calls); + Assert.Equal(2, Fixture.Storage.GetReadCount(JournalId.FromGrainId(grain.GetGrainId()))); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs new file mode 100644 index 00000000000..5c1b4b3cdc0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs @@ -0,0 +1,171 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAcceptanceBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deliver_AcceptedOnlyAfterInboxIsDurableAndWakeupIsScheduled() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var barrier = Fixture.Storage.BlockWrite(journalId); + var envelope = CreateEnvelope(receiver, NewMessage(1, "durability")); + + var delivery = DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + + Assert.False(delivery.IsCompleted); + var staged = Fixture.GetSnapshot(receiver); + Assert.Equal(1, staged.InboxCount); + Assert.Empty(staged.Effects); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())); + + barrier.Release(); + var result = await delivery; + Assert.Equal(DeliveryStatus.Accepted, result.Status); + EndpointSnapshot completed; + try + { + completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Accepted message did not drain. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + Assert.Single(completed.Effects); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(2, Fixture.Storage.GetSuccessfulWriteCount(journalId)); + } + + [Fact] + public async Task ConcurrentDuplicateDeliveries_ConvergeToOneEffectWithinRetention() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/blocked-duplicate"); + var envelope = CreateEnvelope(receiver, NewMessage(11, "duplicate"), "messages/blocked-duplicate"); + + var first = await DeliverAsync(receiver, envelope); + await barrier.WaitUntilEnteredAsync(); + var second = DeliverAsync(receiver, envelope); + + Assert.Equal(DeliveryStatus.Accepted, first.Status); + Assert.False(second.IsCompleted); + Assert.Equal(1, Fixture.GetSnapshot(receiver).InboxCount); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await second).Status); + var state = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(state.Effects).Count); + Assert.Equal(1, state.MaxConcurrentHandlers); + } + + [Fact] + public async Task DuplicateAfterReactivationWithinRetention_RemainsEffectivelyOnce() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope(receiver, NewMessage(13, "reactivation")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForEffectCountAsync(receiver, 1); + await receiver.RequestDeactivationAsync(); + var after = await receiver.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, after.ActivationId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task SameMessageIdFromDistinctSenders_DeduplicatesOneReceiverCommand() + { + var receiver = NewGrain(); + var original = CreateEnvelope(receiver, NewMessage(14, "sender-scoped")); + var other = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("other-sender", "same-message-id")); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, original)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, other)).Status); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task ReorderedDistinctAndDuplicateMessages_ConvergeByApplicationSequence() + { + var receiver = NewGrain(); + var messages = new[] + { + NewMessage(3, "third"), + NewMessage(1, "first"), + NewMessage(2, "second"), + }; + + foreach (var message in messages) + { + var envelope = CreateEnvelope(receiver, message); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + var state = await Fixture.WaitForEffectCountAsync(receiver, 3); + Assert.Equal([1, 2, 3], state.Effects.Select(static effect => effect.Sequence)); + Assert.All(state.Effects, static effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task ConcurrentDelivery_WaitsWhileHandlersRemainSequential() + { + var receiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/sequential"); + var first = CreateEnvelope(receiver, NewMessage(21, "first"), "messages/sequential"); + var second = CreateEnvelope(receiver, NewMessage(22, "second"), "messages/sequential"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await WaitForBarrierAsync(receiver, barrier); + var secondDelivery = DeliverAsync(receiver, second); + Assert.False(secondDelivery.IsCompleted); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + EndpointSnapshot state; + try + { + state = await Fixture.WaitForEffectCountAsync(receiver, 2); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Second message did not complete. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + Assert.Equal(1, state.MaxConcurrentHandlers); + Assert.Equal([21, 22], state.Effects.Select(static effect => effect.Sequence)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs new file mode 100644 index 00000000000..e8a6f82df80 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs @@ -0,0 +1,295 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization.Session; +using Orleans.TestingHost.Diagnostics; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxAdmissionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task ExplicitPreparation_AllowsPriorStateWriteThenCapturesAppliedCohortAtomically() + { + using var attempt = await PrepareAttemptAsync("atomic-admission"); + var state = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(1, state.InboxCount); + Assert.Equal(0, state.ProcessedMessageCount); + Assert.Empty(state.Effects); + Assert.Empty(attempt.Outbox); + await OnTurnAsync(attempt.Context, () => + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "previous-state"); + await attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken); + var priorCapture = attempt.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + Assert.False(attempt.Grain.ApplyAttempted.Task.IsCompleted); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + attempt.Preparation.Release(); + await storage.WaitUntilEnteredAsync(); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var coalesced = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(queued.IsCompleted); + Assert.False(coalesced.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + + var captured = attempt.Grain.GetSnapshotForTest(); + Assert.Equal(0, captured.InboxCount); + Assert.Equal(1, captured.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(captured.Effects).Count); + Assert.Single(attempt.Grain.OutputCaptures[^1]); + Assert.False(queued.IsCompleted); + storage.Release(); + await Task.WhenAll(queued, coalesced); + await attempt.Receiver.RequestDeactivationAsync(); + var recovered = await attempt.Receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task HandlerStorageFailure_DeactivatesAndFailsQueuedWritesAfterPriorAcknowledgement() + { + using var attempt = await PrepareAttemptAsync("fault-ordering"); + IInboxExtension extension = attempt.Context.ActivationServices.GetRequiredService(); + await OnTurnAsync(attempt.Context, () => + { + attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior-captured"; + }); + using var storage = Fixture.Storage.BlockWrite(attempt.JournalId); + var preceding = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await storage.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(attempt.JournalId); + var queued = attempt.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var writes = Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId); + attempt.Preparation.Release(); + await attempt.Grain.ApplyAttempted.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await OnTurnAsync(attempt.Context, () => + { + Assert.Equal(1, attempt.Outbox.SendCalls); + Assert.Single(attempt.Grain.GetSnapshotForTest().Effects); + }); + Assert.False(queued.IsCompleted); + storage.Release(); + await preceding; + + var failure = await Assert.ThrowsAsync(() => queued); + Assert.Same(failure, await attempt.Grain.DeactivationFailure.Task); + await attempt.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + // A failed pump stops the retired activation's inbox; later callers are rerouted to a fresh activation. + await Assert.ThrowsAsync(async () => + await extension.DeliverAsync(attempt.Envelope, TestContext.Current.CancellationToken)); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(attempt.JournalId)); + var failed = attempt.Grain.GetSnapshotForTest(); + Assert.Single(failed.Effects); + Assert.Equal(0, failed.InboxCount); + Assert.Equal(1, failed.ProcessedMessageCount); + _ = await attempt.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(attempt.Receiver, 1); + Assert.NotEqual(failed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + } + + [Fact] + public async Task LateOutgoingIntent_AfterCapture_WaitsForNextCapture() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var grain = Assert.IsType(context.GrainInstance); + var durable = context.ActivationServices.GetRequiredKeyedService>("test-handler-output"); + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var first = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 1); + var late = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 2); + await receiver.StageOutputAsync(first); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var write = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + await OnTurnTaskAsync(context, async () => + { + outbox.Send(late); + }); + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + Assert.Equal(2, outbox.Count); + storage.Release(); + await write; + Assert.Equal(new[] { first.Envelope.MessageId }, grain.OutputCaptures[^1]); + await receiver.RetryWriteStateAsync(); + Assert.Equal(new[] { first.Envelope.MessageId, late.Envelope.MessageId }.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + grain.OutputCaptures[^1].OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + Assert.Equal(2, durable.Count); + await receiver.RequestDeactivationAsync(); + Assert.Equal(2, (await receiver.GetSnapshotAsync()).OutboxCount); + } + + [Fact] + public async Task AcceptanceAfterCapture_IsAcknowledgedByLaterOperation() + { + var receiver = NewGrain(); + var seed = CreateEnvelope(receiver, NewMessage(110, "seed"), "messages/cutoff"); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + + // The pump runs alongside grain calls, so holding the seed's handler keeps it pending in the inbox. + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/cutoff"); + IInboxExtension extension = context.ActivationServices.GetRequiredService(); + Task initial = null!; + await OnTurnAsync(context, () => initial = extension.DeliverAsync(seed, TestContext.Current.CancellationToken).AsTask()); + Assert.Equal(DeliveryStatus.Accepted, (await initial).Status); + await handler.WaitUntilEnteredAsync(); + var grain = Assert.IsType(context.GrainInstance); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + grain.ClearCaptures(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "capture-cutoff"); + var preceding = Fixture.WriteStateAsync(receiver).AsTask(); + await storage.WaitUntilEnteredAsync(); + var incoming = CreateEnvelope(receiver, NewMessage(111, "late-acceptance"), "messages/cutoff"); + Task delivery = null!; + await OnTurnAsync(context, () => delivery = extension.DeliverAsync(incoming).AsTask()); + Assert.False(delivery.IsCompleted); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + storage.Release(); + await preceding; + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + Assert.Equal(1, grain.Captures[0].InboxCount); + Assert.Equal(2, grain.Captures[1].InboxCount); + Assert.Equal(2, grain.GetSnapshotForTest().InboxCount); + handler.Release(); + Assert.Equal(2, (await Fixture.WaitForEffectCountAsync(receiver, 2)).ProcessedMessageCount); + } + + [Fact] + public async Task CallerWaitCancellation_DoesNotWithdrawCapturedAcceptance() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var storage = Fixture.Storage.BlockWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(112, "cancel-wait")); + using var cancellation = new CancellationTokenSource(); + var delivery = DeliverWithCancellationAsync(receiver, envelope, cancellation.Token); + await storage.WaitUntilEnteredAsync(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => delivery); + storage.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task MessagingPrimaryStates_AreTheRegisteredScopedDataInstances() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(receiver).ActivationServices; + var manager = services.GetRequiredService(); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Inbox, out var inbox)); + Assert.Same(services.GetRequiredService(), inbox); + Assert.True(manager.TryGetStateMachine("test-handler-output", out var output)); + Assert.Same(((JournaledTestOutbox)services.GetRequiredService()).StoredMessages, output); + Assert.Same(services.GetRequiredKeyedService>("test-handler-output"), output); + } + + [Fact] + public async Task AcceptanceWriteFailure_DeactivatesAndFreshScopeRetries() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + Fixture.Storage.FailWrite(journal); + var envelope = CreateEnvelope(receiver, NewMessage(114, "acceptance-write-failure")); + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Contains("Injected journal write failure", failure.Message, StringComparison.Ordinal); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var rejected = grain.GetSnapshotForTest(); + Assert.Equal(before.ActivationId, rejected.ActivationId); + Assert.Equal(1, rejected.InboxCount); + Assert.Empty(rejected.Effects); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var deactivation = await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(deactivation); + Assert.Equal(failure.Message, deactivation.Message); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(rejected.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + private async Task PrepareAttemptAsync(string routeSuffix) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var route = "messages/" + routeSuffix; + var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(109, routeSuffix) with { ForwardTo = GrainId.Create("output", routeSuffix) }, route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var preparation = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route + "/application-preparation"); + handler.Release(); + await preparation.WaitUntilEnteredAsync(); + return new(receiver, context, outbox, handler, preparation, envelope); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } + + private static Task OnTurnTaskAsync(IGrainContext context, Func action) + { + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(action()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private sealed record Attempt(IMessagingTestGrain Receiver, IGrainContext Context, JournaledTestOutbox Outbox, + HandlerProbe.Barrier Handler, HandlerProbe.Barrier Preparation, InboxMessage Envelope) : IDisposable + { + public MessagingTestGrain Grain { get; } = Assert.IsType(Context.GrainInstance); + public IJournaledStateManager Manager => Context.ActivationServices.GetRequiredService(); + public JournalId JournalId => JournalId.FromGrainId(Receiver.GetGrainId()); + public void Dispose() { Preparation.Dispose(); Handler.Dispose(); } + } + + private sealed class CallbackContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs new file mode 100644 index 00000000000..b3d66a2d516 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs @@ -0,0 +1,80 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class InboxCapacityCollection : ICollectionFixture +{ + public const string Name = "Messaging inbox capacity"; +} + +[Collection(InboxCapacityCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxCapacityBehaviorTests(InboxCapacityClusterFixture fixture) +{ + [Fact] + public async Task InboxAtCapacity_BackpressuresWithoutPersistenceAndRecoversWhenCapacityFrees() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + var sessions = fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("capacity-test-sender", Guid.NewGuid().ToString("N")); + var poison = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 31, "poison", ThrowDuringPreparation: true)); + var rejected = TestApplicationProtocol.CreateInbox(sessions, sender, "messages/capacity", new TestMessage(TestApplicationProtocol.NewMessageId(), 32, "accepted-after-capacity")); + const string processedInstrument = "orleans-messaging-inbox-messages-processed"; + var firstRetryAcknowledged = fixture.Metrics.WaitForCountAsync(processedInstrument, 1, "retry"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, poison)).Status); + var full = await fixture.WaitForInboxCountAsync(receiver, 1); + Assert.Empty(full.Effects); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, rejected)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + await firstRetryAcknowledged; + Assert.Equal(1, fixture.Metrics.GetCount(processedInstrument, "retry")); + var inbox = fixture.GetInbox(receiver); + var state = await fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, poison.Envelope.MessageId)); + Assert.Equal(1, state.AttemptCount); + var nextAttempt = Assert.IsType(state.NextAttemptAt); + Assert.Equal(fixture.Clock.GetUtcNow() + TimeSpan.FromHours(1), nextAttempt); + fixture.Clock.Advance(TimeSpan.FromHours(2)); + Assert.True(nextAttempt <= fixture.Clock.GetUtcNow()); + var previous = fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, fixture.GetGrainContext(receiver)); + var current = fixture.GetGrainContext(receiver); + var currentGrain = Assert.IsType(current.GrainInstance); + try + { + await fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId == currentGrain.GetSnapshotForTest().ActivationId + && snapshot.InboxDeadLetters.Count == 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Recovered retry did not complete. Activation={snapshot.ActivationId}, inbox={snapshot.InboxCount}, " + + $"processed={snapshot.ProcessedMessageCount}, deadLetters={snapshot.InboxDeadLetters.Count}, " + + $"nextAttempt={nextAttempt:O}, now={fixture.Clock.GetUtcNow():O}, " + + $"failureCompleted={currentGrain.DeactivationFailure.Task.IsCompleted}.", exception); + } + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, rejected)).Status); + var recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("accepted-after-capacity", Assert.Single(recovered.Effects).Value); + Assert.Equal(2, Assert.Single(recovered.InboxDeadLetters).AttemptCount); + } + + private static async Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + await receiver.AsReference().DeliverAsync(envelope); +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs new file mode 100644 index 00000000000..c744619f9a1 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs @@ -0,0 +1,73 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxEnvelopeValidationTests : MessagingBehaviorTestBase +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DefaultMessage_RejectsBeforeAcceptanceOrDuplicate(bool existingKey) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + // External deserialization can materialize invalid default fields before admission. + var malformed = new InboxMessage { Envelope = default }; + var context = Fixture.GetGrainContext(receiver); + if (existingKey) + { + var processedInbox = Fixture.GetInbox(receiver); + await OnTurnAsync(context, () => InboxTestAccess.SetProcessedAt(processedInbox, malformed.Envelope.MessageId, Fixture.Clock.GetUtcNow())); + } + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + + var exception = await Assert.ThrowsAsync(() => DeliverAsync(receiver, malformed)); + + IInboxExtension extension = Fixture.GetInbox(receiver); + await OnTurnAsync(context, () => + { + var direct = extension.DeliverAsync(malformed, TestContext.Current.CancellationToken); + Assert.True(direct.IsCompleted); + Assert.Throws(() => direct.GetAwaiter().GetResult()); + }); + Assert.Contains("message", exception.Message, StringComparison.OrdinalIgnoreCase); + var after = await receiver.GetSnapshotAsync(); + Assert.Equal(0, after.InboxCount); + Assert.Empty(after.Effects); + Assert.Equal(existingKey ? 1 : 0, after.ProcessedMessageCount); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, receiver.GetGrainId())); + } + + [Fact] + public async Task DirectEnvelopeWithExplicitValidIdentity_PreservesDeliveryAndDedupe() + { + var receiver = NewGrain(); + var template = CreateEnvelope(receiver, NewMessage(187, "explicit-identity")); + var envelope = TestApplicationProtocol.With(template, Sessions, + messageId: HierarchicalKey.Parse("tenant/test/command/explicit", null)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); done.SetResult(); } + catch (Exception exception) { done.SetException(exception); } + }); + return done.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs new file mode 100644 index 00000000000..58d131ca819 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -0,0 +1,731 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerCompletionTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task DeadLetter_StagesReasonPayloadAndDedupeBeforeReturnAndPersistsAfterAcknowledgement() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var calls = 0; + handler.Body = (self, _) => + { + calls++; + self.Context.Fail("unsupported input"); + AssertCompletedState(rig, self.Context.Message); + var retained = Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + Assert.Equal("unsupported input", retained.Reason); + Assert.Equal(1, retained.AttemptCount); + Assert.Equal(self.Context.Message.Envelope.MessageId, retained.MessageId); + Assert.Equal(new TestMessage(self.Context.Message.Envelope.MessageId, 501, "async-handler"), + Assert.IsType(TestApplicationProtocol.Read( + rig.Context.ActivationServices.GetRequiredService(), self.Context.Message).Body)); + self.Context.Fail("unsupported input"); + self.Context.Complete(); + Assert.Equal(retained, Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters)); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockAcknowledgement(rig.Journal); + var finished = FinishedAsync(rig); + const string counter = "orleans-messaging-inbox-messages-processed"; + var processed = Fixture.Metrics.WaitForCountAsync(counter, 1, "dead_lettered"); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + Assert.False(finished.IsCompleted); + AssertCompletedState(rig, input); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + storage.Release(); + await WaitAsync(finished); + await processed; + Assert.Equal(1, calls); + Assert.Equal(1, Fixture.Metrics.GetCount(counter, "dead_lettered")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + Assert.Equal(1, calls); + await AssertHealthyAsync(rig); + await DeactivateAsync(rig); + var recovered = await rig.Receiver.GetSnapshotAsync(); + var letter = Assert.Single(recovered.InboxDeadLetters); + Assert.Equal("unsupported input", letter.Reason); + Assert.Equal(1, letter.AttemptCount); + Assert.Equal(input.Envelope.MessageId, letter.MessageId); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + var recoveredContext = Fixture.GetGrainContext(rig.Receiver); + await OnTurnAsync(recoveredContext, async () => + { + var diagnostics = recoveredContext.ActivationServices.GetRequiredService(); + var retained = Assert.Single(diagnostics.InboxDeadLetters); + Assert.Equal(input.Envelope.MessageId, retained.Message.Envelope.MessageId); + Assert.Equal(input.Envelope.Subject, retained.Message.Envelope.Subject); + Assert.Equal(input.Envelope.Payload, retained.Message.Envelope.Payload); + var expectedHeaders = Assert.IsType>(input.Envelope.Headers); + var retainedHeaders = Assert.IsType>(retained.Message.Envelope.Headers); + Assert.Equal(expectedHeaders.Keys.Order(StringComparer.Ordinal), retainedHeaders.Keys.Order(StringComparer.Ordinal)); + foreach (var (name, bytes) in expectedHeaders) + { + Assert.Equal(bytes, retainedHeaders[name]); + } + Assert.True(diagnostics.RemoveInboxDeadLetter(input.Envelope.MessageId)); + await recoveredContext.ActivationServices.GetRequiredService().WriteStateAsync(); + Assert.Empty(diagnostics.InboxDeadLetters); + }); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + + [Collection(MessagingClusterCollection.Name)] + [TestSuite("BVT")] + [TestProvider("None")] + [TestArea("Messaging")] + public sealed class InboxPermanentFailureTests() : MessagingBehaviorTestBase(new PermanentFailureClusterFixture()) + { + [Fact] + public async Task DeadLetter_AfterPreparationRetryTerminatesBeforeLimitAndCountsCurrentAttempt() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var handler = new PermanentHandler(); + var installed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + grain.HandlerOverride = handler; + installed.SetResult(); + }); + await installed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var input = CreateEnvelope(receiver, NewMessage(901, "permanent input"), "permanent/input"); + const string counter = "orleans-messaging-inbox-messages-processed"; + var retried = Fixture.Metrics.WaitForCountAsync(counter, 1, "retry"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, input)).Status); + await retried; + Assert.Equal(1, handler.Attempts); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + var terminated = Fixture.Metrics.WaitForCountAsync(counter, 1, "dead_lettered"); + Fixture.Clock.Advance(TimeSpan.FromHours(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, input)).Status); + await terminated; + var snapshot = await receiver.GetSnapshotAsync(); + Assert.Equal(2, handler.Attempts); + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(1, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.Effects); + var deadLetter = Assert.Single(snapshot.InboxDeadLetters); + Assert.Equal(input.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal("permanent input", deadLetter.Reason); + Assert.Equal(2, deadLetter.AttemptCount); + Assert.Equal(1, Fixture.Metrics.GetCount(counter, "retry")); + Assert.Equal(1, Fixture.Metrics.GetCount(counter, "dead_lettered")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, input)).Status); + Assert.Equal(2, handler.Attempts); + } + + private sealed class PermanentHandler : IInboxHandler + { + public int Attempts { get; private set; } + + public bool CanHandle(IInboxHandlerContext context) => true; + + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (++Attempts == 1) + { + throw new ArgumentException("preparation failed"); + } + context.Fail("permanent input"); + return ValueTask.CompletedTask; + } + } + + private sealed class PermanentFailureClusterFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 5; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + options.DeduplicationWindow = TimeSpan.FromDays(7); + } + } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task DeadLetter_InvalidReasonLeavesTerminalStateUnchanged(string? reason) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + var error = Assert.ThrowsAny(() => self.Context.Fail(reason!)); + Assert.Equal("reason", error.ParamName); + Assert.Single(rig.Inbox); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + self.Mutate(); + self.Context.Complete(); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeadLetter_ConflictingTerminalOutcomeRejectsAttemptAndDeactivates(bool completeFirst) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + Exception? expected = null; + handler.Body = (self, _) => + { + if (completeFirst) self.Context.Complete(); + else self.Context.Fail("original reason"); + expected = Assert.Throws(() => self.Context.Fail("different reason")); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + handler.Release.TrySetResult(); + var failure = await WaitAsync(rig.Grain.DeactivationFailure.Task); + Assert.Same(expected, failure); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + await AssertFailureReplayAsync(rig, input); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeadLetter_HandlerErrorAfterFailPreservesTerminalOutcomeUnlessStorageFails(bool failWrite) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var handlerError = new IOException("after dead letter"); + handler.Body = (self, _) => + { + self.Context.Fail("permanent input failure"); + throw handlerError; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + AssertCompletedState(rig, input); + Assert.Equal("permanent input failure", Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters).Reason); + var storageError = new IOException("actual persistence failure"); + if (failWrite) storage.Fail(storageError); + else storage.Release(); + await WaitAsync(finished); + if (failWrite) + { + Assert.Same(storageError, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + Assert.NotEqual("permanent input failure", Assert.Single((await rig.Receiver.GetSnapshotAsync()).InboxDeadLetters).Reason); + } + else + { + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + await AssertHealthyAsync(rig); + } + } + + [Fact] + public async Task Complete_StagesBusinessOutputAndDedupeBeforeHandlerReturn() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, token) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, self.Context.Message); + Assert.Equal(1, Assert.Single(rig.Effects).Value.Count); + Assert.Single(rig.Outbox); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockAcknowledgement(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + await DeactivateAsync(rig); + var recovered = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + + [Fact] + public async Task Complete_PrecedingWriterCapturesCompletionWithBusinessAcrossReturnContinuation() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var preceding = OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + await storage.WaitUntilEnteredAsync(); + var priorCapture = rig.Grain.Captures[^1]; + Assert.Empty(priorCapture.Effects); + Assert.Equal(1, priorCapture.InboxCount); + Assert.Equal(0, priorCapture.ProcessedMessageCount); + Assert.Equal(0, priorCapture.OutboxCount); + var queued = OnTurnAsync(rig.Context, async () => await rig.Manager.WriteStateAsync(CancellationToken.None)); + var staged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var returnContinuation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.Body = async (self, _) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + AssertCompletedState(rig, input); + staged.TrySetResult(); + await returnContinuation.Task; + }; + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + try + { + await WaitAsync(staged.Task); + Assert.False(finished.IsCompleted); + storage.Release(); + await WaitAsync(Task.WhenAll(preceding, queued)); + Assert.False(finished.IsCompleted); + Assert.Equal(writes + 2, Writes(rig)); + var capturedEffects = Assert.Single(rig.Grain.Captures, snapshot => snapshot.Effects.Count != 0); + Assert.Equal(0, capturedEffects.InboxCount); + Assert.Equal(1, capturedEffects.ProcessedMessageCount); + Assert.Equal(1, capturedEffects.OutboxCount); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(capturedEffects.Effects)); + } + finally + { + returnContinuation.TrySetResult(); + } + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 1); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerFailure_BeforeVsAfterCompleteUsesActualLogicalOutcome(bool completed) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var error = new IOException("Handler outcome."); + handler.Body = (self, _) => + { + if (completed) + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + } + throw error; + }; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + if (completed) + { + AssertSuccess(rig, input, outputCount: 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, input)).Status); + } + else + { + Assert.Empty(rig.Effects); + Assert.Empty(rig.Outbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Grain.GetSnapshotForTest().InboxDeadLetters).MessageId); + } + await AssertHealthyAsync(rig); + await DeactivateAsync(rig); + var replay = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(completed ? 1 : 0, replay.Effects.Count); + Assert.Equal(1, replay.ProcessedMessageCount); + Assert.Equal(completed ? 0 : 1, replay.InboxDeadLetters.Count); + } + + [Fact] + public async Task SuccessfulReturnWithoutComplete_IsExplicitTerminalMisuse() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (_, _) => ValueTask.CompletedTask; + var input = await DeliverAsync(rig); + var writes = Writes(rig); + handler.Release.TrySetResult(); + var error = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Contains("must call Complete", error.Message, StringComparison.Ordinal); + Assert.Equal(writes, Writes(rig)); + await rig.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Inbox).Key); + Assert.Equal(input.Envelope.Payload.ToArray(), Assert.Single(rig.Inbox).Value.Message.Envelope.Payload.ToArray()); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Effects); + await AssertFailureReplayAsync(rig, input); + } + + [Fact] + public async Task RepeatedComplete_SameActiveAttemptStagesExactlyOnce() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + self.Mutate(); + self.Context.Complete(); + var timestamp = Assert.Single(rig.Processed).Value; + self.Context.Complete(); + Assert.Equal(timestamp, Assert.Single(rig.Processed).Value); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + [InlineData("dead-letter")] + public async Task RetainedContext_AfterRetirementRejectsAndKeepsOwnerHealthy(string operation) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + var error = await Assert.ThrowsAsync(() => + OnTurnAsync(rig.Context, () => InvokeContextOperation(handler, operation))); + Assert.Contains("inactive or different attempt", error.Message, StringComparison.Ordinal); + AssertSuccess(rig, input, outputCount: 0); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData("complete")] + [InlineData("dead-letter")] + public async Task RetainedContext_DuringOtherAttemptThrowsWithoutAffectingCurrentAttempt(string operation) + { + var rig = await CreateAsync(); + using var first = rig.Handler; + var one = await DeliverAsync(rig); + var finished = FinishedAsync(rig); + first.Release.TrySetResult(); + await WaitAsync(finished); + using var second = new TestHandler(rig.Effects, rig.Context); + InvalidOperationException? rejection = null; + second.Body = (_, _) => + { + try { InvokeContextOperation(first, operation); } + catch (InvalidOperationException error) { rejection = error; } + throw new IOException("Replacement failure."); + }; + await OnTurnAsync(rig.Context, () => + rig.Grain.HandlerOverride = second); + var two = CreateEnvelope(rig.Receiver, NewMessage(502, "next"), "async/next"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, two)).Status); + await WaitAsync(second.Entered.Task); + second.Release.TrySetResult(); + var snapshot = await Fixture.WaitForDeadLetterCountAsync(rig.Receiver, 1); + Assert.Contains("inactive or different attempt", Assert.IsType(rejection).Message, StringComparison.Ordinal); + var letter = Assert.Single(snapshot.InboxDeadLetters); + Assert.Equal(two.Envelope.MessageId, letter.MessageId); + Assert.Equal("Replacement failure.", letter.Reason); + Assert.Equal(one.Envelope.MessageId, Assert.Single(rig.Effects).Key); + await AssertHealthyAsync(rig); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletionWrite_StorageOutcomeDecidesCommit(bool failWrite) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Body = (self, _) => + { + self.Mutate(); + rig.Outbox.Send(self.Output); + self.Context.Complete(); + return ValueTask.CompletedTask; + }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + AssertCompletedState(rig, input); + Assert.False(finished.IsCompleted); + var failure = new OperationCanceledException("Actual storage canceled.", new CancellationToken(canceled: true)); + if (failWrite) storage.Fail(failure); + else storage.Release(); + await WaitAsync(finished); + if (failWrite) + { + Assert.Same(failure, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + else + { + AssertSuccess(rig, input, outputCount: 1); + await AssertHealthyAsync(rig); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletionWrite_HookOutcomePreservesActualCommit(bool postCommit) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var input = await DeliverAsync(rig); + var error = new IOException("Journal hook failed."); + await OnTurnAsync(rig.Context, () => + { + if (postCommit) rig.Grain.AfterJournalOperationFailure = error; + else rig.Grain.BeforeJournalOperationFailure = error; + }); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await WaitAsync(finished); + if (postCommit) + { + AssertSuccess(rig, input, outputCount: 0); + await OnTurnAsync(rig.Context, () => rig.Grain.AfterJournalOperationFailure = null); + await AssertHealthyAsync(rig); + } + else + { + var failure = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Same(error, failure.InnerException); + await AssertFailureReplayAsync(rig, input); + } + } + + [Fact] + public async Task PostCompleteHandlerFailure_WithStorageFailureKeepsStorageCauseAuthoritative() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var handlerError = new InvalidOperationException("After Complete."); + handler.Body = (self, _) => { self.Mutate(); self.Context.Complete(); throw handlerError; }; + var input = await DeliverAsync(rig); + using var storage = Fixture.Storage.BlockWrite(rig.Journal); + var finished = FinishedAsync(rig); + handler.Release.TrySetResult(); + await storage.WaitUntilEnteredAsync(); + var storageError = new IOException("Storage failed."); + storage.Fail(storageError); + await WaitAsync(finished); + Assert.Same(storageError, await WaitAsync(rig.Grain.DeactivationFailure.Task)); + await AssertFailureReplayAsync(rig, input); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var handler = new TestHandler(effects, context); + var rig = new Rig(receiver, context, Assert.IsType(context.GrainInstance), + services.GetRequiredService(), (JournaledTestOutbox)services.GetRequiredService(), + effects, handler, services.GetRequiredService()); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private async Task DeliverAsync(Rig rig) + { + var input = CreateEnvelope(rig.Receiver, NewMessage(501, "async-handler"), "async/handler"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, input)).Status); + await WaitAsync(rig.Handler.Entered.Task); + return input; + } + + private static void InvokeContextOperation(TestHandler handler, string operation) + { + switch (operation) + { + case "complete": handler.Context.Complete(); break; + case "dead-letter": handler.Context.Fail("unusable input"); break; + default: throw new ArgumentException(nameof(operation)); + } + } + + private static void AssertCompletedState(Rig rig, InboxMessage input) + { + Assert.Empty(rig.Inbox); + Assert.Equal(input.Envelope.MessageId, Assert.Single(rig.Processed).Key); + } + + private static void AssertSuccess(Rig rig, InboxMessage input, int outputCount) + { + AssertCompletedState(rig, input); + Assert.Equal(new MessageEffect(input.Envelope.MessageId, 1, 501, "async-handler"), Assert.Single(rig.Effects).Value); + Assert.Equal(outputCount, rig.Outbox.Count); + Assert.Empty(rig.Grain.GetSnapshotForTest().InboxDeadLetters); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(rig.Journal); + private static async Task AssertHealthyAsync(Rig rig) + { + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + private static async Task DeactivateAsync(Rig rig) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(rig.Context.Deactivated); + } + + private async Task AssertFailureReplayAsync(Rig rig, InboxMessage input) + { + await WaitAsync(rig.Context.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForDeadLetterCountAsync(rig.Receiver, 1); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(input.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + /// + /// Completes after the pump finishes its current pass (including the batch write) and one more pass. + /// + private Task FinishedAsync(Rig rig) + { + return RunAsync(); + + async Task RunAsync() + { + try + { + await Fixture.RunPumpAsync(rig.Receiver); + } + catch (OperationCanceledException) when (rig.Context.ActivationServices.GetRequiredService().ShutdownToken.IsCancellationRequested) + { + // A failed write stops the pump; callers assert the failure explicitly. + } + } + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception error) + { + done.SetException(error); + } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, MessagingTestGrain Grain, + IJournaledStateManager Manager, JournaledTestOutbox Outbox, IDurableDictionary Effects, + TestHandler Handler, Inbox InboxState) + { + public IReadOnlyDictionary Inbox => InboxTestAccess.GetPendingMessages(InboxState); + public IReadOnlyDictionary Processed => InboxTestAccess.GetProcessedMessages(InboxState); + + public JournalId Journal => JournalId.FromGrainId(Receiver.GetGrainId()); + } + + private sealed class TestHandler(IDurableDictionary effects, IGrainContext grainContext) : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public IInboxHandlerContext Context { get; private set; } = null!; + public OutboxMessage Output { get; private set; } + public Func Body { get; set; } = + static (self, _) => { self.Mutate(); self.Context.Complete(); return default; }; + public bool CanHandle(IInboxHandlerContext context) => true; + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + Context = context; + + Output = TestApplicationProtocol.Create(grainContext.ActivationServices.GetRequiredService(), grainContext.GrainId, grainContext.GrainId, "output", 41); + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + await Body(this, cancellationToken); + } + public void Mutate() + { + effects.TryGetValue(Context.Message.Envelope.MessageId, out var previous); + effects[Context.Message.Envelope.MessageId] = + new MessageEffect(Context.Message.Envelope.MessageId, (previous?.Count ?? 0) + 1, 501, "async-handler"); + } + public void Dispose() + { + Release.TrySetResult(); + + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs new file mode 100644 index 00000000000..4a0c937b01f --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs @@ -0,0 +1,154 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHandlerTransactionTests : MessagingBehaviorTestBase +{ + public InboxHandlerTransactionTests() : base(receiverOnly: true) + { + } + + [Fact] + public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndStagedOutputAtomically() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var logicalId = TestApplicationProtocol.NewMessageId(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(logicalId, 7, "atomic", sink.GetGrainId())); + + var result = await DeliverAsync(receiver, envelope); + var receiverState = await Fixture.WaitForEffectCountAsync(receiver, 1); + var output = Fixture.GetStagedOutput(receiver); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + var effect = Assert.Single(receiverState.Effects); + Assert.Equal(new MessageEffect(logicalId, 1, 7, "atomic"), effect); + Assert.Equal(0, receiverState.InboxCount); + Assert.Equal(1, receiverState.OutboxCount); + var outgoing = Assert.Single(output); + Assert.Equal(sink.GetGrainId(), outgoing.ReceiverId); + Assert.Equal("messages/forwarded", TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Route); + var body = TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), outgoing).Body; + Assert.Equal(new TestMessage(logicalId, 7, "atomic"), body); + Assert.Equal(1, receiverState.ProcessedMessageCount); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(receiverState.ActivationId, recovered.ActivationId); + Assert.Equal(effect, Assert.Single(recovered.Effects)); + Assert.Equal(1, recovered.OutboxCount); + Assert.Equal(outgoing.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + + var duplicate = await DeliverAsync(receiver, envelope); + Assert.Equal(DeliveryStatus.Duplicate, duplicate.Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + Assert.Single(Fixture.GetStagedOutput(receiver)); + } + + [Fact] + public async Task HandlerPreparationFailure_DeadLettersWithoutStagingEffectsOrOutput() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); + + var accepted = await DeliverAsync(receiver, envelope); + var state = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, accepted.Status); + Assert.Empty(state.Effects); + Assert.Equal(0, state.InboxCount); + Assert.Equal(0, state.OutboxCount); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal(1, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(state.ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(recovered.InboxDeadLetters).MessageId); + } + + [Fact] + public async Task HandlerPreparation_OrdinaryWriteLeavesEffectsLocalUntilApplyAndCompletion() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var envelope = CreateEnvelope(receiver, NewMessage(10, "premature-commit") with { CommitDuringHandling = true }); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal("premature-commit", Assert.Single(completed.Effects).Value); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(oldGrain.DeactivationFailure.Task.IsCompleted); + Assert.Same(oldContext, Fixture.GetGrainContext(receiver)); + } + + [Fact] + public async Task HandlerCompletionWriteFailure_FencesOldManagerAndFreshActivationRetries() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/completion-failure"); + var envelope = CreateEnvelope(receiver, NewMessage(78, "completion-failure"), "messages/completion-failure"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync(() => oldManager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != oldGrain.GetSnapshotForTest().ActivationId + && snapshot.Effects.Sum(static effect => effect.Count) == 1); + Assert.NotEqual(oldGrain.GetSnapshotForTest().ActivationId, completed.ActivationId); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task PreparationFailureAccountingWriteFailure_RecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failure-accounting"); + var envelope = CreateEnvelope(receiver, NewMessage(79, "failure-accounting") with { ThrowDuringPreparation = true }, "messages/failure-accounting"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal(1, Assert.Single(completed.InboxDeadLetters).AttemptCount); + Assert.Empty(completed.Effects); + Assert.Empty(Fixture.GetStagedOutput(receiver)); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs new file mode 100644 index 00000000000..6bc7c26a737 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -0,0 +1,260 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxHeaderBehaviorTests : MessagingBehaviorTestBase +{ + public InboxHeaderBehaviorTests() : base(new HeaderFixture()) { } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RawMessage_AbsentOptionalMetadataCommitsAndDeduplicates(bool empty) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + byte[] payload = empty ? [] : [0, 255, 128, 42]; + var message = Raw(HierarchicalKey.Create("raw", Guid.NewGuid().ToString("N")), payload); + Assert.Null(message.Envelope.Subject); + Assert.Null(message.Envelope.Headers); + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + using var storage = Fixture.Storage.BlockAcknowledgement(JournalId.FromGrainId(context.GrainId)); + var delivery = DeliverAsync(receiver, message); + await storage.WaitUntilEnteredAsync(); + Assert.False(delivery.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + storage.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await delivery).Status); + await WaitAsync(handler.Entered.Task); + Assert.Equal(payload, handler.Message.Envelope.Payload.ToArray()); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(message.Envelope.MessageId, Assert.Single(completed.Effects).LogicalId); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + await receiver.RequestDeactivationAsync(); + await WaitAsync(context.Deactivated); + var replayed = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, replayed.ActivationId); + Assert.Equal(1, Assert.Single(replayed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + } + + [Fact] + public async Task PendingDuplicate_ChangedCustomHeadersPreservesOriginal() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var key = HierarchicalKey.Create("custom", Guid.NewGuid().ToString("N")); + var original = Raw(key, [1, 2, 3], "original"); + var changed = Raw(key, [1, 2, 3], "changed"); + using var handler = new RawHandler(context); + await OnTurnAsync(context, () => grain.HandlerOverride = handler); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var pending = context.ActivationServices.GetRequiredService(); + Envelope first = default; + await OnTurnAsync(context, () => first = Assert.Single(pending.Messages()).Envelope); + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + var schedules = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, context.GrainId); + await OnTurnAsync(context, async () => + { + IInboxExtension extension = context.ActivationServices.GetRequiredService(); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(changed)).Status); + var retained = Assert.Single(pending.Messages()).Envelope; + Assert.Same(first.Payload, retained.Payload); + Assert.Same(first.Headers, retained.Headers); + Assert.True(Assert.IsType>(retained.Headers).TryGetValue("custom.trace", out var trace)); + Assert.Equal("original", Encoding.UTF8.GetString(trace)); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(schedules, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, context.GrainId)); + handler.Release.TrySetResult(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + [Fact] + public async Task UnknownHeaders_RecoverAndRemainAvailableAfterRetryAndDeadLetter() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("headers", Guid.NewGuid().ToString("N")); + var body = NewMessage(810, "poison") with { LogicalId = key, ThrowDuringPreparation = true }; + var application = CreateEnvelope(receiver, body); + byte[] opaque = [0xff, 0, 0x80, 0x7f]; + var headers = new Dictionary(Assert.IsType>(application.Envelope.Headers), + StringComparer.Ordinal) + { + ["future.opaque"] = opaque, + ["future.number"] = new Serializer(Sessions).SerializeToArray(8675309) + }; + var message = application with { Envelope = application.Envelope with { Headers = headers } }; + var retry = Fixture.Metrics.WaitForCountAsync("orleans-messaging-inbox-messages-processed", 1, "retry"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await retry; + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(previous.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var current = Fixture.GetGrainContext(receiver); + Assert.NotSame(previous, current); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.InboxDeadLetters); + await OnTurnAsync(current, () => + { + var stored = Assert.Single(current.ActivationServices.GetRequiredService().Messages()).Envelope; + var metadata = Assert.IsType>(stored.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); + }); + Fixture.Clock.Advance(TimeSpan.FromHours(1)); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + current = Fixture.GetGrainContext(receiver); + var dead = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Equal(key, Assert.Single(dead.InboxDeadLetters).MessageId); + Assert.Equal(2, Assert.Single(dead.InboxDeadLetters).AttemptCount); + Assert.Empty(dead.Effects); + var diagnostics = current.ActivationServices.GetRequiredService(); + await OnTurnAsync(current, () => + { + var retained = Assert.Single(diagnostics.InboxDeadLetters).Message.Envelope; + var metadata = Assert.IsType>(retained.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); + Assert.Equal(application.Envelope.Payload.ToArray(), retained.Payload.ToArray()); + }); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + _ = await receiver.GetSnapshotAsync(); + var recovered = Fixture.GetGrainContext(receiver); + await OnTurnAsync(recovered, () => + { + var retained = Assert.Single(recovered.ActivationServices.GetRequiredService() + .InboxDeadLetters).Message.Envelope; + var metadata = Assert.IsType>(retained.Headers); + Assert.True(metadata.TryGetValue("future.opaque", out var bytes)); + Assert.Equal(opaque, bytes); + Assert.True(metadata.TryGetValue("future.number", out var number)); + Assert.Equal(8675309, new Serializer(Sessions).Deserialize(number)); + }); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, Raw(key, []))).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InvalidMessageHeader_RejectsBeforeSchedulingOrMutation(bool completed) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var key = HierarchicalKey.Create("malformed", Guid.NewGuid().ToString("N")); + var message = Raw(key, [1, 2]); + if (completed) + { + await OnTurnAsync(context, () => + InboxTestAccess.SetProcessedAt(context.ActivationServices.GetRequiredService(), key, Fixture.Clock.GetUtcNow())); + } + message = message with + { + Envelope = message.Envelope with + { + Headers = new(StringComparer.Ordinal) { ["invalid"] = null! } + } + }; + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); + await OnTurnAsync(context, async () => + { + IInboxExtension extension = context.ActivationServices.GetRequiredService(); + await Assert.ThrowsAsync(() => extension.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); + }); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, context.GrainId)); + var snapshot = await receiver.GetSnapshotAsync(); + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(completed ? 1 : 0, snapshot.ProcessedMessageCount); + Assert.Empty(snapshot.Effects); + } + + private static InboxMessage Raw(HierarchicalKey key, byte[] payload, string? trace = null) + { + Dictionary? headers = null; + if (trace is not null) (headers ??= new(StringComparer.Ordinal))["custom.trace"] = Encoding.UTF8.GetBytes(trace); + return new() { Envelope = new() { MessageId = key, Payload = payload, Headers = headers } }; + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var result = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = Run(); }); + return result.Task; + async Task Run() + { + try { await action(); result.SetResult(); } + catch (Exception exception) { result.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => + task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + private sealed class RawHandler(IGrainContext context) : IInboxHandler, IDisposable + { + public InboxMessage Message { get; private set; } + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool CanHandle(IInboxHandlerContext context) => true; + + + public async ValueTask HandleAsync(IInboxHandlerContext handler, CancellationToken cancellationToken) + { + Message = handler.Message; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + var key = handler.Message.Envelope.MessageId; + var effects = context.ActivationServices.GetRequiredKeyedService>("test-effects"); + effects[key] = new(key, 1, 800, "raw"); + handler.Complete(); + } + + public void Dispose() => Release.TrySetResult(); + } + + private sealed class HeaderFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs new file mode 100644 index 00000000000..ba7f2eee8c5 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -0,0 +1,462 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxIdentityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingCommand_NewSenderCoalescesAndPreservesOriginalEnvelope() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1042", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = TestApplicationProtocol.With(original, Sessions, sender: GrainId.Create("forwarder", "second")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId); + byte[] storedPayload = null!; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Message.Envelope.Payload); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, repeated)).Status); + await OnTurnAsync(rig.Context, () => + { + Assert.True(rig.Inbox.MessageState.TryGetValue(key, out var entry)); + var stored = entry.Message; + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, stored.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(stored.Envelope)); + Assert.Same(storedPayload, stored.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), stored.Envelope.Payload.ToArray()); + Assert.Single(rig.Pending); + Assert.Empty(rig.Processed); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(key, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).LogicalId); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData("subject")] + [InlineData("subject-case")] + [InlineData("body")] + public async Task PendingCommand_ConflictingContentRejectsBeforeSchedulingOrMutation(string difference) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1043", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var conflictingSubject = difference switch + { + "subject" => "inventory.release.v1", + "subject-case" => "Inventory.reserve.v1", + _ => Assert.IsType(TestApplicationProtocol.Subject(original.Envelope)) + }; + var replacement = Create(rig, key, conflictingSubject, "different-parameters", + GrainId.Create("forwarder", "other")); + var conflict = difference == "body" ? replacement : TestApplicationProtocol.With(original, Sessions, + sender: TestApplicationProtocol.Sender(Sessions, replacement.Envelope), subject: conflictingSubject); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId); + byte[] storedPayload = null!; + await OnTurnAsync(rig.Context, () => storedPayload = Assert.Single(rig.Pending).Value.Message.Envelope.Payload); + var failure = await Assert.ThrowsAsync(() => DeliverOnTurnAsync(rig, conflict)); + Assert.Contains(key.ToString(), failure.Message, StringComparison.Ordinal); + Assert.Contains("different command", failure.Message, StringComparison.Ordinal); + await OnTurnAsync(rig.Context, () => + { + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(rig.Pending).Value.Message.Envelope)); + Assert.Equal(TestApplicationProtocol.Subject(original.Envelope), TestApplicationProtocol.Subject(Assert.Single(rig.Pending).Value.Message.Envelope)); + Assert.Same(storedPayload, Assert.Single(rig.Pending).Value.Message.Envelope.Payload); + Assert.Equal(original.Envelope.Payload.ToArray(), Assert.Single(rig.Pending).Value.Message.Envelope.Payload.ToArray()); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + }); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + await OnTurnAsync(rig.Context, async () => + { + rig.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "healthy"; + await rig.Manager.WriteStateAsync(CancellationToken.None); + }); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, original)).Status); + handler.Release.TrySetResult(); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, handler.Calls); + Assert.Equal(1, Assert.Single(rig.Grain.GetSnapshotForTest().Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletedCommand_ChangedSenderSubjectAndBodyAcknowledgesWithoutFingerprint(bool restart) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1044", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "completely.changed.subject.v2", "different-body", + GrainId.Create("forwarder", "new-owner")); + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.WaitForInboxCountAsync(rig.Receiver, 0); + var prior = rig.Context; + if (restart) + { + await rig.Receiver.RequestDeactivationAsync(); + await WaitAsync(prior.Deactivated); + await rig.Receiver.GetSnapshotAsync(); + Assert.NotSame(prior, Fixture.GetGrainContext(rig.Receiver)); + } + var writes = Writes(rig); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + } + + [Fact] + public async Task ExactParentChildAndSiblingCommandsHaveIndependentCompletions() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var parent = HierarchicalKey.Create("tenant", "orders", "1045"); + var keys = new[] { parent, parent.CreateChildKey("reserve"), parent.CreateChildKey("release") }; + for (var index = 0; index < keys.Length; index++) + { + var command = Create(rig, keys[index], "inventory.command.v1", $"command-{index}"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, command)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, index + 1); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, command)).Status); + } + var completed = await rig.Receiver.GetSnapshotAsync(); + Assert.Equal(3, completed.Effects.Count); + Assert.Equal(3, completed.ProcessedMessageCount); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(keys.OrderBy(static key => key.ToString(), StringComparer.Ordinal), + completed.Effects.Select(static effect => effect.LogicalId).OrderBy(static key => key.ToString(), StringComparer.Ordinal)); + } + + [Fact] + public async Task SameCommandKey_InDifferentReceiversCompletesIndependently() + { + var first = await CreateAsync(); + var second = await CreateAsync(); + using var firstHandler = first.Handler; + using var secondHandler = second.Handler; + firstHandler.Release.TrySetResult(); + secondHandler.Release.TrySetResult(); + var key = HierarchicalKey.Create("tenant", "orders", "1049", "reserve"); + var one = Create(first, key, "inventory.reserve.v1", "first-receiver"); + var two = Create(second, key, "inventory.reserve.v1", "second-receiver"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(second.Receiver, two)).Status); + var firstOutcome = await Fixture.WaitForEffectCountAsync(first.Receiver, 1); + var secondOutcome = await Fixture.WaitForEffectCountAsync(second.Receiver, 1); + Assert.Equal(key, Assert.Single(firstOutcome.Effects).LogicalId); + Assert.Equal(key, Assert.Single(secondOutcome.Effects).LogicalId); + Assert.Equal(1, firstOutcome.ProcessedMessageCount); + Assert.Equal(1, secondOutcome.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(first.Receiver, one)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(second.Receiver, two)).Status); + Assert.Equal(1, firstHandler.Calls); + Assert.Equal(1, secondHandler.Calls); + } + + [Fact] + public async Task PendingIdentityAfterReplayStillRejectsConflictAndDeduplicatesNewSender() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "orders", "1046", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await WaitAsync(handler.Entered.Task); + await OnTurnAsync(rig.Context, () => + { + rig.Context.Deactivate(new DeactivationReason( + DeactivationReasonCode.ApplicationRequested, "Verify pending command identity replay."), + TestContext.Current.CancellationToken); + }); + await WaitAsync(rig.Context.Deactivated); + using var recoveredHandler = new IdentityHandler(); + using var read = Fixture.Storage.BlockRead(JournalId.FromGrainId(rig.Context.GrainId)); + var activation = rig.Receiver.GetSnapshotAsync(); + await read.WaitUntilEnteredAsync(); + var recoveredContext = Fixture.GetGrainContext(rig.Receiver); + var recoveredGrain = Assert.IsType(recoveredContext.GrainInstance); + await OnTurnAsync(recoveredContext, () => recoveredGrain.HandlerOverride = recoveredHandler); + read.Release(); + await WaitAsync(recoveredHandler.Entered.Task); + var recovered = CreateRig(rig.Receiver, recoveredContext, recoveredHandler); + Assert.NotSame(rig.Context, recovered.Context); + var repeated = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "after-replay")); + var conflict = Create(recovered, key, "inventory.reserve.v1", "changed", + GrainId.Create("forwarder", "after-replay")); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(recovered, repeated)).Status); + await Assert.ThrowsAsync(() => DeliverOnTurnAsync(recovered, conflict)); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, Assert.Single(recovered.Pending).Value.Message.Envelope)); + Assert.Empty(recovered.Processed); + Assert.False(recovered.Grain.DeactivationFailure.Task.IsCompleted); + recoveredHandler.Release.TrySetResult(); + await activation; + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + } + + [Fact] + public async Task DeadLetterInspectionAndRemovalUseReceiverLocalKeyAcrossReplay() + { + var receiver = NewGrain(); + var key = HierarchicalKey.Create("tenant", "orders", "1048", "invalid-command"); + var body = new TestMessage(key, 611, "poison", ThrowDuringPreparation: true); + var original = CreateEnvelope(receiver, body); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); + var deadLetter = Assert.Single((await Fixture.WaitForDeadLetterCountAsync(receiver, 1)).InboxDeadLetters); + Assert.Equal(key, deadLetter.MessageId); + var old = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(old.Deactivated); + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(key, Assert.Single(recovered.InboxDeadLetters).MessageId); + var duplicate = TestApplicationProtocol.With(original, Sessions, + sender: GrainId.Create("forwarder", "deadletter"), subject: "another.subject.v1"); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + Assert.True(await receiver.RemoveInboxDeadLetterAsync(key)); + Assert.False(await receiver.RemoveInboxDeadLetterAsync(key)); + var current = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await WaitAsync(current.Deactivated); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, duplicate)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MissingOptionalMetadata_RawHandlerCompletesWithoutSubject(bool removeSender) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = HierarchicalKey.Create("tenant", "invalid-subject", "command"); + var envelope = Create(rig, key, "valid.subject.v1", "missing-subject"); + var raw = TestApplicationProtocol.With(envelope, Sessions, removeSubject: true, removeSender: removeSender); + var bytes = envelope.Envelope.Payload.ToArray(); + handler.Release.TrySetResult(); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverOnTurnAsync(rig, raw)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Null(raw.Envelope.Subject); + if (removeSender) Assert.DoesNotContain("sender", Assert.IsType>(raw.Envelope.Headers)); + Assert.Equal(bytes, raw.Envelope.Payload.ToArray()); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverOnTurnAsync(rig, raw)).Status); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + public async Task OversizedMetadata_RejectsBeforeSchedulingOrPersistence(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + var key = field switch + { + "key-bytes" => HierarchicalKey.Create(new string('x', 1025)), + "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 513)), + _ => HierarchicalKey.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 257), + "subject-utf8" => new string('\u00e9', 129), + _ => "inventory.reserve.v1" + }; + var writes = Writes(rig); + var message = Create(rig, key, subject, "oversized"); + await Assert.ThrowsAnyAsync(() => DeliverOnTurnAsync(rig, message)); + Assert.Equal(writes, Writes(rig)); + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, rig.Context.GrainId)); + Assert.Empty(rig.Pending); + Assert.Empty(rig.Processed); + Assert.Empty(rig.Grain.GetSnapshotForTest().Effects); + Assert.False(rig.Grain.DeactivationFailure.Task.IsCompleted); + Assert.False(handler.Entered.Task.IsCompleted); + } + + [Theory] + [InlineData("key-bytes")] + [InlineData("key-utf8")] + [InlineData("subject-bytes")] + [InlineData("subject-utf8")] + [InlineData("subject-space")] + public async Task MetadataAtExactDefaultLimit_IsAccepted(string field) + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = field switch + { + "key-bytes" => HierarchicalKey.Create(new string('x', 1024)), + "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 512)), + "key-depth" => HierarchicalKey.Create(Enumerable.Repeat("segment", 32).ToArray()), + _ => HierarchicalKey.Create("tenant", "bounded", "command") + }; + var subject = field switch + { + "subject-bytes" => new string('x', 256), + "subject-utf8" => new string('\u00e9', 128), + "subject-space" => " ", + _ => "inventory.reserve.v1" + }; + var envelope = Create(rig, key, subject, "maximum"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, envelope)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, envelope)).Status); + } + + [Fact] + public async Task ExpiredIdentityCanAcceptChangedSenderSubjectAndBodyAtExactBoundary() + { + var rig = await CreateAsync(); + using var handler = rig.Handler; + handler.Release.TrySetResult(); + var key = HierarchicalKey.Create("tenant", "orders", "1047", "reserve"); + var original = Create(rig, key, "inventory.reserve.v1", "original"); + var repeated = Create(rig, key, "inventory.reserve.v2", "changed", + GrainId.Create("forwarder", "new")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); + await Fixture.WaitForEffectCountAsync(rig.Receiver, 1); + await Fixture.WaitForInboxCountAsync(rig.Receiver, 0); + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(rig.Receiver, repeated)).Status); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, repeated)).Status); + var completed = await Fixture.WaitForEffectCountAsync(rig.Receiver, 2); + Assert.Equal(2, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(2, handler.Calls); + } + + private async Task CreateAsync() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var handler = new IdentityHandler(); + var rig = CreateRig(receiver, context, handler); + await OnTurnAsync(context, () => rig.Grain.HandlerOverride = handler); + return rig; + } + + private static Rig CreateRig(IMessagingTestGrain receiver, IGrainContext context, IdentityHandler handler) => + new(receiver, context, Assert.IsType(context.GrainInstance), handler, + context.ActivationServices.GetRequiredService(), + context.ActivationServices.GetRequiredService()); + + private InboxMessage Create(Rig rig, HierarchicalKey key, string subject, string value, GrainId sender = default) => + TestApplicationProtocol.CreateInbox(Sessions, + sender.IsDefault ? GrainId.Create("producer", "first") : sender, + subject, new TestMessage(key, 610, value), key); + + private int Writes(Rig rig) => Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(rig.Context.GrainId)); + + private static Task DeliverOnTurnAsync(Rig rig, InboxMessage envelope) + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Context.Scheduler.QueueAction(() => + { + try + { + Assert.Same(rig.Context, ReceiverTestServices.CurrentGrainContext); + IInboxExtension extension = rig.Context.ActivationServices.GetRequiredService(); + started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); + } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) => + OnTurnAsync(context, () => { action(); return Task.CompletedTask; }); + + private static Task OnTurnAsync(IGrainContext context, Func action) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => { _ = CompleteAsync(); }); + return done.Task; + async Task CompleteAsync() + { + try + { + Assert.Same(context, ReceiverTestServices.CurrentGrainContext); + await action(); + done.SetResult(); + } + catch (Exception exception) { done.SetException(exception); } + } + } + + private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, + MessagingTestGrain Grain, IdentityHandler Handler, IJournaledStateManager Manager, Inbox Inbox) + { + public IReadOnlyDictionary Pending => InboxTestAccess.GetPendingMessages(Inbox); + public IReadOnlyDictionary Processed => InboxTestAccess.GetProcessedMessages(Inbox); + } + + private sealed class IdentityHandler : IInboxHandler, IDisposable + { + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public int Calls { get; private set; } + public bool CanHandle(IInboxHandlerContext context) => true; + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + var services = ReceiverTestServices.CurrentGrainContext!.ActivationServices; + var effects = services.GetRequiredKeyedService>("test-effects"); + var application = TestApplicationProtocol.Read(services.GetRequiredService(), context.Message); + var body = Assert.IsType(application.Body); + Calls++; + Entered.TrySetResult(); + await Release.Task.WaitAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + effects.TryGetValue(context.Message.Envelope.MessageId, out var prior); + effects[context.Message.Envelope.MessageId] = new MessageEffect(context.Message.Envelope.MessageId, + (prior?.Count ?? 0) + 1, body.Sequence, body.Value); + context.Complete(); + } + public void Dispose() => Release.TrySetResult(); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs new file mode 100644 index 00000000000..f85dce575de --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxJournalRecoveryTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task FailedInboxAcceptance_FencesOldObjectsAndFreshReplayOmitsEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(2, "failed-acceptance")); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.NotSame(oldManager, Fixture.GetGrainContext(receiver).ActivationServices.GetRequiredService()); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.Effects); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + + [Fact] + public async Task AmbiguousInboxAcceptanceCommit_PreservesAndProcessesRecoveredEnvelope() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + Fixture.Storage.FailAfterWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var envelope = CreateEnvelope(receiver, NewMessage(76, "ambiguous-acceptance")); + + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal("ambiguous-acceptance", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task CompletionWriteFailure_RecoversInFreshActivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + const string route = "messages/inbox-completion-retry"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(57, "inbox-completion-retry"), route); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); + handler.Release(); + + var cleaned = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.InboxCount == 0 && snapshot.Effects.Count > 0); + + Assert.Equal(1, Assert.Single(cleaned.Effects).Count); + Assert.Empty(cleaned.InboxDeadLetters); + } + + [Fact] + public async Task WakeupSchedulingFailure_RetriedDeliveryProcessesOnce() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(61, "schedule-failure")); + Fixture.JobManagerProbe.FailNext(MessagingPump.JobName); + + // The wakeup is scheduled while the acceptance write is prepared, so its failure prevents that commit. + var failure = await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.IsType(failure.InnerException); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + var status = (await DeliverAsync(receiver, envelope)).Status; + Assert.True(status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate, status.ToString()); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs new file mode 100644 index 00000000000..31bb572d302 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs @@ -0,0 +1,57 @@ +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxLifecycleBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task Deactivation_StopsPumpAndWaitsForActiveHandler() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var shutdown = Fixture.GetPump(receiver).ShutdownToken; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/teardown-active"); + var envelope = CreateEnvelope(receiver, NewMessage(95, "active-handler"), "messages/teardown-active"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var stopping = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = shutdown.Register(stopping.SetResult); + + context.Deactivate(new(DeactivationReasonCode.ApplicationRequested, "Verify active handler teardown."), TestContext.Current.CancellationToken); + + await stopping.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.False(context.Deactivated.IsCompleted); + handler.Release(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } + + [Fact] + public async Task DeactivationBeforeProcessing_RecoveryDrainsAcceptedMessageOnce() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(96, "queued")); + + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs new file mode 100644 index 00000000000..8c357572a72 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxQuiescenceTests.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxQuiescenceTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task OwnerDelete_AfterAcknowledgedWork_StopsPumpAndUsesFreshActivation() + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/delete-ack"); + var first = CreateEnvelope(receiver, NewMessage(160, "first"), "messages/delete-ack"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, first)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var pump = Fixture.GetPump(receiver); + var outbox = GetOutbox(context); + var second = CreateEnvelope(receiver, NewMessage(161, "second"), "messages/delete-ack"); + var secondDelivery = DeliverAsync(receiver, second); + + handler.Release(); + Assert.Equal(DeliveryStatus.Accepted, (await secondDelivery).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(2, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + using var storage = Fixture.Storage.BlockDelete(JournalId.FromGrainId(receiver.GetGrainId())); + var deletion = receiver.DeleteStateAndDeactivateAsync(); + await storage.WaitUntilEnteredAsync(); + Assert.True(outbox.Stopping.IsCompleted); + Assert.True(pump.ShutdownToken.IsCancellationRequested); + Assert.False(deletion.IsCompleted); + storage.Release(); + await deletion; + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var deleted = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, deleted.ActivationId); + Assert.Empty(deleted.Effects); + Assert.Equal(0, deleted.ProcessedMessageCount); + } + + [Fact] + public async Task OwnerDelete_DiscardsStagedOutputAndLaterDeliveryUsesFreshActivation() + { + var receiver = NewGrain(); + await receiver.StageEffectAsync(new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 190, "delete-this")); + await receiver.RetryWriteStateAsync(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = GetOutbox(context); + var staged = TestApplicationProtocol.Create(Sessions, receiver.GetGrainId(), receiver.GetGrainId(), + "output/staged", NewMessage(193, "staged-output")); + { + await receiver.StageOutputAsync(staged); + Assert.Equal(staged.Envelope.MessageId, Assert.Single(outbox.Messages).Envelope.MessageId); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + await receiver.DeleteStateAndDeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(outbox.Messages); + Assert.Empty(grain.GetSnapshotForTest().Effects); + Assert.True(outbox.Stopping.IsCompleted); + Assert.Throws(() => outbox.Send(staged)); + } + Assert.Empty(outbox.Messages); + var fresh = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, fresh.ActivationId); + Assert.Empty(fresh.Effects); + Assert.Equal(0, fresh.OutboxCount); + var envelope = CreateEnvelope(receiver, NewMessage(191, "new-state")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var after = await Fixture.WaitForEffectCountAsync(receiver, 1); + _ = await receiver.GetSnapshotAsync(); + Assert.Equal(fresh.ActivationId, after.ActivationId); + Assert.Equal("new-state", Assert.Single(after.Effects).Value); + Assert.Equal(1, after.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + } + + private static JournaledTestOutbox GetOutbox(IGrainContext context) => + (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs new file mode 100644 index 00000000000..98d1ec2bc91 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -0,0 +1,134 @@ +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxRetryTests() : MessagingBehaviorTestBase(new RetryFixture()) +{ + private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + + [Fact] + public async Task HandlerFailure_RetriesAfterBackoffAndEarlyWakeupsAreHarmless() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(192, "retry") with { ThrowOnceDuringPreparation = true }); + var messageId = envelope.Envelope.MessageId; + var now = Fixture.Clock.GetUtcNow(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); + var state = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId)); + Assert.Equal(now + RetryDelay, state.NextAttemptAt); + Assert.Contains("Injected handler preparation failure", state.LastError); + + // Wake-ups before the retry is due must not run the handler. + var early = await Fixture.RunPumpAsync(receiver); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, early.Status); + _ = await Fixture.RunPumpAsync(receiver); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.Effects); + + Fixture.Clock.Advance(RetryDelay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Empty(completed.InboxDeadLetters); + + // Spurious wake-ups with no work complete immediately and change nothing. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task OrdinaryHandlerFailures_DeadLetterAtExactConfiguredAttemptLimit() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(193, "always-fails") with { ThrowDuringPreparation = true }); + var messageId = envelope.Envelope.MessageId; + var start = Fixture.Clock.GetUtcNow(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); + + // Backoff doubles per attempt. + Fixture.Clock.Advance(RetryDelay); + _ = await Fixture.RunPumpAsync(receiver); + await Fixture.WaitUntilAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 2); + var second = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId)); + Assert.Equal(start + RetryDelay + RetryDelay * 2, second.NextAttemptAt); + + Fixture.Clock.Advance(RetryDelay * 2); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var deadLettered = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + var deadLetter = Assert.Single(deadLettered.InboxDeadLetters); + Assert.Equal(messageId, deadLetter.MessageId); + Assert.Equal(3, deadLetter.AttemptCount); + Assert.Equal(0, deadLettered.InboxCount); + Assert.Empty(deadLettered.Effects); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(deadLetter, Assert.Single(recovered.InboxDeadLetters)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Fact] + public async Task AcceptedMessage_WaitsForHandlerRegistrationWithoutDeadLettering() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(194, "awaits-handler"), "messages/awaits-handler"); + using (var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/awaits-handler")) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await WaitForBarrierAsync(receiver, barrier); + await receiver.ConfigureHandlerAsync(false); + + // Deactivation cancels the in-flight handler without consuming an attempt. + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + var withoutHandler = await receiver.GetSnapshotAsync(); + Assert.Equal(1, withoutHandler.InboxCount); + Assert.Empty(withoutHandler.Effects); + Assert.Empty(withoutHandler.InboxDeadLetters); + Assert.NotEqual(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + + await receiver.ConfigureHandlerAsync(true); + var noHandlerContext = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await noHandlerContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + _ = await Fixture.RunPumpAsync(receiver); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("awaits-handler", Assert.Single(completed.Effects).Value); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + private sealed class RetryFixture : MessagingClusterFixture + { + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxProcessingAttempts = 3; + options.BackpressureRetryDelay = RetryDelay; + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs new file mode 100644 index 00000000000..fc8f6a2d9ed --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs @@ -0,0 +1,99 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateEncodingTests : MessagingBehaviorTestBase +{ + public InboxStateEncodingTests() : base(new FaultingInboxCodecFixture()) { } + + [Fact] + public async Task CompleteCommandCodecFailure_PreservesOriginalCauseAndReplaysAcknowledgedState() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/complete-codec-fault"); + var envelope = CreateEnvelope(receiver, NewMessage(204, "complete-codec-fault"), "messages/complete-codec-fault"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Completion remove command failed."); + ((FaultingInboxCodecFixture)Fixture).NextFailure = failure; + handler.Release(); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(outbox.Messages); + _ = await receiver.GetSnapshotAsync(); + var fresh = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + var replayed = Assert.IsType(fresh.ReplayedSnapshot); + Assert.Equal(1, replayed.InboxCount); + Assert.Equal(0, replayed.ProcessedMessageCount); + Assert.Empty(replayed.Effects); + Assert.Equal(0, replayed.OutboxCount); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InboxCodecFailure_PreventsStorageAndRetriedDeliveryProcessesOnce(bool snapshot) + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Injected inbox command codec failure."); + var faulting = (FaultingInboxCodecFixture)Fixture; + faulting.NextFailure = failure; + faulting.FailOnSnapshot = snapshot; + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + var envelope = CreateEnvelope(receiver, NewMessage(203, "codec")); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, envelope)); + Assert.Equal(snapshot, faulting.FailedSnapshot); + Assert.Empty(outbox.Messages); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + if (snapshot) + { + // The snapshot fails while the write is encoded, after the acceptance was applied, so the activation is discarded. + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(1, grain.GetSnapshotForTest().InboxCount); + Assert.Empty(grain.GetSnapshotForTest().Effects); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.Effects); + Assert.Equal(0, recovered.ProcessedMessageCount); + } + else + { + // Commands are encoded before they are applied, so a rejected acceptance leaves the activation's state unchanged. + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + var current = await receiver.GetSnapshotAsync(); + Assert.Equal(before.ActivationId, current.ActivationId); + Assert.Equal(0, current.InboxCount); + Assert.Empty(current.Effects); + Assert.Equal(0, current.ProcessedMessageCount); + } + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs new file mode 100644 index 00000000000..13b7358565c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateManagerBoundaryTests.cs @@ -0,0 +1,228 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Journaling.Json; +using Orleans.Messaging.Tests.Support; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateManagerBoundaryTests +{ + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ActivationStandardState_UsesScopedCodecAndSingleOwnerEnrollment(bool keyedFirst) + { + const string selectedFormat = "orleans-binary"; + var builder = CreateBuilder(selectedFormat); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedScoped>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + builder.Services.AddScoped(); + var grainType = GrainType.Create("codec-scope"); + var manifest = new GrainManifest( + ImmutableDictionary.Empty.Add( + grainType, new GrainProperties( + ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal, StringComparer.Ordinal))), + ImmutableDictionary.Empty); + var manifestProvider = Substitute.For(); + manifestProvider.LocalGrainManifest.Returns(manifest); + manifestProvider.Current.Returns(new ClusterManifest( + new MajorMinorVersion(1, 0), + ImmutableDictionary.Empty, + [manifest])); + builder.Services.AddSingleton(manifestProvider); + builder.Services.AddSingleton(); + builder.Services.AddScoped(sp => + { + var context = Substitute.For(); + context.ActivationServices.Returns(sp); + context.GrainId.Returns(GrainId.Create(grainType, IdSpan.Create(Guid.NewGuid().ToString("N")))); + context.ObservableLifecycle.Returns(Substitute.For()); + return context; + }); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + IDurableDictionary state; + OwnedComponent component; + Dependency dependency; + IJournaledStateManager journalOwner; + await using (var scope = services.CreateAsyncScope()) + { + var sp = scope.ServiceProvider; + Assert.Same(manifest.Grains[grainType], + sp.GetRequiredService().GetGrainProperties(sp.GetRequiredService().GrainId.Type)); + var app = sp.GetRequiredService(); + journalOwner = sp.GetRequiredService(); + Assert.Same(app, journalOwner); + var codec = Assert.IsType(sp.GetRequiredKeyedService>(selectedFormat)); + Assert.Throws(() => services.GetRequiredKeyedService>(selectedFormat)); + state = keyedFirst + ? sp.GetRequiredKeyedService>("state") + : app.GetOrAddState>("state"); + component = sp.GetRequiredService(); + dependency = sp.GetRequiredService(); + Assert.Same(state, component.State); + Assert.Same(dependency, component.Dependency); + Assert.Same(state, sp.GetRequiredKeyedService>("state")); + Assert.Same(state, app.GetOrAddState>("state")); + Assert.True(app.TryGetState>("state", out var found)); + Assert.Same(state, found); + Assert.True(journalOwner.TryGetStateMachine("state", out var machine)); + Assert.Same(state, machine); + sp.GetRequiredService().ObservableLifecycle.Received(1) + .Subscribe(Arg.Any(), GrainLifecycleStage.SetupState, Arg.Any()); + await journalOwner.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("committed", 17); + Assert.Equal(1, codec.SetsWritten); + await app.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + Assert.Equal(17, state["committed"]); + Assert.Throws(() => app.GetOrAddState>("late")); + Assert.Throws(() => sp.GetRequiredKeyedService>("late")); + Assert.Same(state, app.GetOrAddState>("state")); + await journalOwner.DeleteStateAsync(Cancellation); + Assert.Empty(state); + Assert.False(dependency.Disposed); + Assert.Equal(0, component.Disposals); + await using var other = services.CreateAsyncScope(); + var otherOwner = other.ServiceProvider.GetRequiredService(); + Assert.NotSame(journalOwner, otherOwner); + Assert.NotSame(codec, other.ServiceProvider.GetRequiredKeyedService>(selectedFormat)); + } + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + await Assert.ThrowsAsync(() => journalOwner.WriteStateAsync(Cancellation).AsTask()); + } + + [Fact] + public async Task NamedStandaloneOwner_UsesSelectedFormatAndCallerOwnedComponentsAcrossReplay() + { + const string selectedFormat = "json"; + const string storageName = "caller-owned-store"; + var builder = CreateBuilder(selectedFormat); + builder.AddVolatileJournalStorage(storageName); + builder.UseJsonJournalFormat(StandardStateJsonContext.Default); + var codecType = GetDictionaryCodecType(builder.Services, selectedFormat); + builder.Services.AddKeyedSingleton>(selectedFormat, (sp, _) => + new TrackingCodec((IDurableDictionaryCommandCodec)ActivatorUtilities.CreateInstance(sp, codecType))); + var dependenciesCreated = 0; + builder.Services.AddScoped(_ => { dependenciesCreated++; return new Dependency(); }); + var id = new JournalId("inbox-state-lifetime/" + Guid.NewGuid().ToString("N")); + builder.Services.AddScoped(sp => + sp.GetRequiredKeyedService(storageName).CreateStandalone(id)); + builder.Services.AddScoped(sp => new OwnedComponent( + sp.GetRequiredKeyedService>("state"), sp.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + Assert.Equal(0, dependenciesCreated); + var codec = Assert.IsType(services.GetRequiredKeyedService>(selectedFormat)); + Dependency dependency; + OwnedComponent component; + await using (var scope = services.CreateAsyncScope()) + { + var first = scope.ServiceProvider.GetRequiredService(); + Assert.False(first is IDurableStateManager); + dependency = scope.ServiceProvider.GetRequiredService(); + Assert.False(first.TryGetStateMachine("state", out _)); + component = scope.ServiceProvider.GetRequiredService(); + var state = component.State; + Assert.Same(scope.ServiceProvider.GetRequiredKeyedService>("state"), state); + Assert.True(first.TryGetStateMachine("state", out var registered)); + Assert.Same(state, registered); + await first.InitializeAsync(Cancellation); + Assert.Equal(0, codec.Applies); + state.Add("persisted", 23); + await first.WriteStateAsync(Cancellation); + Assert.Equal(1, codec.SetsWritten); + await first.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + await using var recoveryScope = services.CreateAsyncScope(); + var recovered = recoveryScope.ServiceProvider.GetRequiredService(); + Assert.NotSame(first, recovered); + Assert.False(recovered is IDurableStateManager); + var recoveredCodec = recoveryScope.ServiceProvider.GetRequiredKeyedService>(selectedFormat); + Assert.Same(codec, recoveredCodec); + var fresh = recoveryScope.ServiceProvider.GetRequiredKeyedService>("state"); + await recovered.InitializeAsync(Cancellation); + Assert.NotSame(state, fresh); + Assert.Equal(new KeyValuePair("persisted", 23), Assert.Single(fresh)); + Assert.Equal(1, codec.Applies); + await recovered.DeleteStateAsync(Cancellation); + Assert.Empty(fresh); + await recovered.DisposeAsync(); + Assert.Equal(0, component.Disposals); + Assert.False(dependency.Disposed); + Assert.Equal(1, dependenciesCreated); + } + + Assert.True(dependency.Disposed); + Assert.Equal(1, component.Disposals); + } + + private static Type GetDictionaryCodecType(IServiceCollection services, string selectedFormat) => services.Last(entry => + entry.ServiceType == typeof(IDurableDictionaryCommandCodec<,>) && Equals(entry.ServiceKey, selectedFormat)) + .KeyedImplementationType!.MakeGenericType(typeof(string), typeof(int)); + + internal static TestSiloBuilder CreateBuilder(string selectedFormat) + { + var builder = new TestSiloBuilder(); + builder.Services.AddSerializer(); + builder.Services.AddLogging(); + builder.Services.AddSingleton(TimeProvider.System); + builder.Services.AddKeyedSingleton(KeyedService.AnyKey, static (sp, _) => sp.GetRequiredService()); + builder.AddVolatileJournalStorage(); + builder.Services.Configure(options => options.JournalFormatKey = selectedFormat); + return builder; + } + + internal sealed class TestSiloBuilder : ISiloBuilder + { + public IServiceCollection Services { get; } = new ServiceCollection(); + public IConfiguration Configuration { get; } = new ConfigurationBuilder().Build(); + } + + private sealed class Dependency : IDisposable + { + public bool Disposed { get; private set; } + public void Dispose() => Disposed = true; + } + + private sealed class OwnedComponent(IDurableDictionary state, Dependency dependency) : IDisposable + { + public IDurableDictionary State { get; } = state; + public Dependency Dependency { get; } = dependency; + public int Disposals { get; private set; } + public void Dispose() => Disposals++; + } + + private sealed class TrackingCodec(IDurableDictionaryCommandCodec inner) : IDurableDictionaryCommandCodec + { + public int SetsWritten { get; private set; } + public int Applies { get; private set; } + public void WriteSet(string key, int value, JournalStreamWriter writer) { SetsWritten++; inner.WriteSet(key, value, writer); } + public void WriteRemove(string key, JournalStreamWriter writer) => inner.WriteRemove(key, writer); + public void WriteClear(JournalStreamWriter writer) => inner.WriteClear(writer); + public void WriteSnapshot(IReadOnlyCollection> items, JournalStreamWriter writer) => inner.WriteSnapshot(items, writer); + public void Apply(JournalBufferReader input, IDurableDictionaryCommandHandler consumer) + { + Applies++; + inner.Apply(input, consumer); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs new file mode 100644 index 00000000000..cb2c412ba1d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateProtocolTests.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxStateProtocolTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task RawWriteDuringLocalPreparation_CapturesNoHandlerEffects() + { + var receiver = NewGrain(); + const string route = "messages/local-preparation"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(201, "local"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + await OnTurnAsync(context, () => context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "raw-write"); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.False(grain.ApplyAttempted.Task.IsCompleted); + var beforeApply = grain.GetSnapshotForTest(); + Assert.Empty(beforeApply.Effects); + Assert.Equal(1, beforeApply.InboxCount); + Assert.Equal(0, beforeApply.ProcessedMessageCount); + var captured = grain.Captures[^1]; + Assert.Empty(captured.Effects); + Assert.Equal(1, captured.InboxCount); + Assert.Equal(0, captured.ProcessedMessageCount); + handler.Release(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + await receiver.RequestDeactivationAsync(); + var replayed = await receiver.GetSnapshotAsync(); + Assert.Equal(completed.Effects, replayed.Effects); + Assert.Equal(1, replayed.ProcessedMessageCount); + } + + [Fact] + public async Task HandlerFailureBeforeComplete_AccountsPreparationOutcomeWithoutMutationInference() + { + var receiver = NewGrain(); + const string route = "messages/no-earlier-write"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(203, "no-earlier-write"), route); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Local preparation failed before Complete."); + await OnTurnAsync(context, () => + { + Assert.Equal(0, manager.PendingWriteByteCount); + grain.NextApplyFailure = failure; + }); + + handler.Release(); + var completed = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Contains(failure.Message, Assert.Single(completed.InboxDeadLetters).Reason, StringComparison.Ordinal); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Empty(completed.Effects); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.False(grain.DeactivationFailure.Task.IsCompleted); + await receiver.RequestDeactivationAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(grain.GetSnapshotForTest().ActivationId, recovered.ActivationId); + Assert.Empty(recovered.Effects); + Assert.Equal(completed.InboxDeadLetters, recovered.InboxDeadLetters); + Assert.Equal(1, recovered.ProcessedMessageCount); + Assert.Equal(0, recovered.InboxCount); + } + + private static Task OnTurnAsync(IGrainContext context, Action action) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { action(); completion.SetResult(); } + catch (Exception exception) { completion.SetException(exception); } + }); + return completion.Task; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs new file mode 100644 index 00000000000..ab280b2198d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/JournaledTestOutboxBehaviorTests.cs @@ -0,0 +1,122 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JournaledTestOutboxBehaviorTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task EquivalentDuplicates_BeforeAndAfterCommit_PreserveOneOutput() + { + var owner = NewGrain(); + var original = CreateOutput(owner); + var copied = original; + var serializer = Fixture.Client.ServiceProvider.GetRequiredService>(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + Assert.Same(original.Envelope.Payload, copied.Envelope.Payload); + Assert.Same(original.Envelope.Headers, copied.Envelope.Headers); + Assert.NotSame(original.Envelope.Payload, copy.Envelope.Payload); + Assert.NotSame(original.Envelope.Headers, copy.Envelope.Headers); + + await owner.StageOutputAsync(original); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + await owner.StageOutputAsync(copied); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); + await owner.RetryWriteStateAsync(); + await owner.StageOutputAsync(copy); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); + var before = await owner.GetSnapshotAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + await owner.StageOutputAsync(copy); + + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + Assert.Equal(1, recovered.OutboxCount); + } + + [Theory] + [InlineData("receiver")] + [InlineData("body-bytes")] + public async Task ConflictingDuplicate_RejectsChangedEnvelopeAndPreservesCommittedOutput(string difference) + { + var owner = NewGrain(); + var original = CreateOutput(owner); + await owner.StageOutputAsync(original); + await owner.RetryWriteStateAsync(); + var retained = Assert.Single(Fixture.GetStagedOutput(owner)); + var conflicting = difference switch + { + "receiver" => original with { ReceiverId = GrainId.Create("other-receiver", "1") }, + _ => original with + { + Envelope = TestApplicationProtocol.With(original.ToInboxMessage(), Sessions, + payload: CreateOutput(owner, changed: true).Envelope.Payload).Envelope + } + }; + + var exception = await Assert.ThrowsAsync(() => owner.StageOutputAsync(conflicting)); + + Assert.Contains(original.Envelope.MessageId.ToString(), exception.Message, StringComparison.Ordinal); + Assert.Same(retained.Envelope.Headers, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Headers); + Assert.Same(retained.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(owner)).Envelope.Payload); + await owner.RetryWriteStateAsync(); + await owner.RequestDeactivationAsync(); + var recovered = await owner.GetSnapshotAsync(); + Assert.Equal(1, recovered.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(owner)); + Assert.Equal(original.Envelope.MessageId, output.Envelope.MessageId); + Assert.Equal(TestApplicationProtocol.Sender(Sessions, original.Envelope), TestApplicationProtocol.Sender(Sessions, output.Envelope)); + Assert.Equal(original.ReceiverId, output.ReceiverId); + Assert.Equal(0, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), output).Body); + } + + [Fact] + public async Task HandlerEquivalentDuplicateOutput_CommitsOneEffectAndOneOutput() + { + var receiver = NewGrain(); + var target = GrainId.Create("output-target", "handler"); + var envelope = CreateEnvelope( + receiver, + NewMessage(97, "duplicate-output") with { ForwardTo = target }, + "messages/duplicate-output"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + static snapshot => snapshot.InboxCount == 0 && (snapshot.Effects.Count > 0 || snapshot.InboxDeadLetters.Count > 0)); + + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(1, completed.OutboxCount); + var output = Assert.Single(Fixture.GetStagedOutput(receiver)); + Assert.Equal(target, output.ReceiverId); + await receiver.RequestDeactivationAsync(); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(completed.ActivationId, recovered.ActivationId); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(output.Envelope.MessageId, Assert.Single(Fixture.GetStagedOutput(receiver)).Envelope.MessageId); + Assert.Empty(recovered.InboxDeadLetters); + } + + + + + + private OutboxMessage CreateOutput(IMessagingTestGrain owner, bool changed = false) => + TestApplicationProtocol.Create(Fixture.Client.ServiceProvider.GetRequiredService(), + owner.GetGrainId(), owner.GetGrainId(), "output", changed ? 1 : 0); +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs new file mode 100644 index 00000000000..39e60402481 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingClusterCollection.cs @@ -0,0 +1,9 @@ +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class MessagingClusterCollection +{ + public const string Name = "Messaging cluster"; +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs new file mode 100644 index 00000000000..8bf6e751ecb --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs @@ -0,0 +1,262 @@ +using System.Diagnostics.Metrics; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingMetricCardinalityTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task PendingAndProcessedDuplicates_RecordOneBoundedReceiptEach() + { + var receiver = NewGrain(); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var services = context.ActivationServices; + IInboxExtension extension = services.GetRequiredService(); + // The pump runs alongside grain calls, so holding the handler keeps the accepted message pending. + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/metric-pending"); + var envelope = CreateEnvelope(receiver, NewMessage(451, "duplicate-receipts"), "messages/metric-pending"); + Assert.Equal(DeliveryStatus.Accepted, (await StartDelivery()).Status); + await handler.WaitUntilEnteredAsync(); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId())); + var pending = Fixture.GetSnapshot(receiver); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.ProcessedMessageCount); + Assert.Empty(pending.Effects); + using var probe = new MessageMetricListener(services.GetRequiredService().Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + + Assert.Equal(DeliveryStatus.Duplicate, (await StartDelivery()).Status); + Assert.Single(probe.Read("inbox-messages-received")).AssertCounter(grainType, "duplicate"); + handler.Release(); + var processed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(0, processed.InboxCount); + Assert.Equal(1, processed.ProcessedMessageCount); + Assert.Equal(1, Assert.Single(processed.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + var receipts = probe.Read("inbox-messages-received"); + Assert.Equal(2, receipts.Length); + Assert.All(receipts, receipt => receipt.AssertCounter(grainType, "duplicate")); + Assert.Single(receipts.Select(receipt => receipt.Series).Distinct()); + + Task StartDelivery() + { + var started = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try { started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); } + catch (Exception exception) { started.SetException(exception); } + }); + return started.Task.Unwrap(); + } + } + + [Theory] + [InlineData(1)] + [InlineData(64)] + public async Task DistinctApplicationRoutes_KeepFixedMetricSeriesAndMissingHandlerOutcomes(int routeCount) + { + var receiver = NewGrain(); + var missingHandler = NewGrain(); + await missingHandler.ConfigureHandlerAsync(false); + await ReactivateAsync(missingHandler); + await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + using var probe = new MessageMetricListener(context.ActivationServices.GetRequiredService().Meter); + var grainType = receiver.GetGrainId().Type.ToString(); + var routes = new HashSet(); + for (var index = 0; index < routeCount; index++) + { + var unknown = $"unknown/{Guid.NewGuid():N}"; + routes.Add(unknown); + var rejected = CreateEnvelope(missingHandler, NewMessage(index, "unknown"), unknown); + Assert.Equal(DeliveryStatus.HandlerNotFound, (await DeliverAsync(missingHandler, rejected)).Status); + Assert.Equal(unknown, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), rejected).Route); + var valid = $"messages/cardinality/{Guid.NewGuid():N}"; + routes.Add(valid); + var accepted = CreateEnvelope(receiver, NewMessage(index, "accepted"), valid); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, accepted)).Status); + Assert.Equal(valid, TestApplicationProtocol.Read(Fixture.Client.ServiceProvider.GetRequiredService(), accepted).Route); + await Fixture.WaitForEffectCountAsync(receiver, index + 1); + } + + await probe.WaitForCountAsync("inbox-processing-duration", routeCount); + var receivedRows = probe.Read("inbox-messages-received"); + Assert.Equal(routeCount * 2, routes.Count); + Assert.Equal(routeCount * 2, receivedRows.Length); + Assert.Equal(2, receivedRows.Select(row => row.Series).Distinct().Count()); + var acceptedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "accepted"))).ToArray(); + var rejectedRows = receivedRows.Where(row => row.Tags.Any(tag => tag.Key == "status" && Equals(tag.Value, "handler_not_found"))).ToArray(); + Assert.Equal(routeCount, acceptedRows.Length); + Assert.Equal(routeCount, rejectedRows.Length); + Assert.All(acceptedRows, row => row.AssertCounter(grainType, "accepted")); + Assert.All(rejectedRows, row => row.AssertCounter(grainType, "handler_not_found")); + var processed = probe.Read("inbox-messages-processed"); + Assert.Equal(routeCount, processed.Length); + Assert.Single(processed.Select(row => row.Series).Distinct()); + Assert.All(processed, row => row.AssertCounter(grainType, "success")); + var durations = probe.Read("inbox-processing-duration"); + Assert.Equal(routeCount, durations.Length); + Assert.Single(durations.Select(row => row.Series).Distinct()); + Assert.All(durations, row => + { + Assert.Equal("ms", row.Unit); + Assert.True(row.Value >= 0); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, row.Tags); + }); + var completed = await receiver.GetSnapshotAsync(); + Assert.Equal(routeCount, completed.Effects.Count); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + Assert.Equal(routeCount, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Empty(completed.InboxDeadLetters); + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingInstrumentTagTests +{ + [Theory] + [InlineData("inbox-messages-received", "accepted")] + [InlineData("inbox-messages-processed", "success")] + public void Counters_EmitExactBoundedTags(string instrumentName, string? status) + { + using var factory = new TestMeterFactory(); + var instruments = new Orleans.Messaging.MessagingInstruments(new OrleansInstruments(factory)); + using var probe = new MessageMetricListener(factory.Meter); + const string grainType = "instrument-test"; + Action emit = instrumentName switch + { + "inbox-messages-received" => () => instruments.OnInboxMessageReceived(grainType, status!), + _ => () => instruments.OnInboxMessageProcessed(grainType, status!), + }; + emit(); + var measurement = Assert.Single(probe.Read(instrumentName)); + Assert.Equal(MessageMetricListener.Prefix + instrumentName, measurement.Name); + measurement.AssertCounter(grainType, status); + } + + [Theory] + [InlineData("inbox-processing-duration")] + public void Durations_RecordMillisecondsAndClampNegativeValues(string instrumentName) + { + using var factory = new TestMeterFactory(); + var instruments = new Orleans.Messaging.MessagingInstruments(new OrleansInstruments(factory)); + using var probe = new MessageMetricListener(factory.Meter); + const string grainType = "instrument-test"; + Action record = duration => instruments.OnInboxProcessingDuration(duration, grainType); + record(TimeSpan.FromMilliseconds(12.5)); + record(TimeSpan.FromMilliseconds(-10)); + var values = probe.Read(instrumentName); + Assert.Equal(2, values.Length); + Assert.All(values, value => + { + Assert.Equal("ms", value.Unit); + Assert.Equal(new[] { new KeyValuePair("grain_type", grainType) }, value.Tags); + }); + Assert.Equal(12.5, values[0].Value); + Assert.Equal(0, values[1].Value); + } + + [Fact] + public void DepthInstruments_RecordNonZeroDeltasTaggedByGrainType() + { + using var factory = new TestMeterFactory(); + var instruments = new Orleans.Messaging.MessagingInstruments(new OrleansInstruments(factory)); + using var probe = new MessageMetricListener(factory.Meter); + instruments.OnInboxDepthChanged("instrument-test", 3); + instruments.OnInboxDepthChanged("instrument-test", 0); + instruments.OnInboxDepthChanged("instrument-test", -1); + var inboxDepth = probe.Read("inbox-depth"); + Assert.Equal([3d, -1d], inboxDepth.Select(static row => row.Value)); + Assert.All(inboxDepth, row => + { + Assert.Null(row.Unit); + Assert.Equal(new[] { new KeyValuePair("grain_type", "instrument-test") }, row.Tags); + }); + } + private sealed class TestMeterFactory : IMeterFactory + { + public Meter Meter { get; private set; } = null!; + public Meter Create(MeterOptions options) => Meter = new Meter(options); + public void Dispose() => Meter.Dispose(); + } +} + +internal sealed class MessageMetricListener : IDisposable +{ + internal const string Prefix = "orleans-messaging-"; + private readonly MeterListener _listener; + private readonly object _lock = new(); + private readonly List _measurements = []; + private readonly List<(string Name, int Count, TaskCompletionSource Completion)> _waiters = []; + + public MessageMetricListener(Meter meter) + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (ReferenceEquals(instrument.Meter, meter) && instrument.Name.StartsWith(Prefix, StringComparison.Ordinal)) + listener.EnableMeasurementEvents(instrument); + } + }; + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.Start(); + } + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) + { + lock (_lock) + { + _measurements.Add(new(instrument.Name, instrument.Unit, value, tags.ToArray())); + foreach (var waiter in _waiters.ToArray()) + { + if (_measurements.Count(row => row.Name == waiter.Name) >= waiter.Count) + { + _waiters.Remove(waiter); + waiter.Completion.TrySetResult(); + } + } + } + } + public Task WaitForCountAsync(string name, int count) + { + lock (_lock) + { + if (_measurements.Count(row => row.Name == Prefix + name) >= count) return Task.CompletedTask; + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((Prefix + name, count, completion)); + return completion.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + } + public Measurement[] Read(string name) + { + lock (_lock) return _measurements.Where(row => row.Name == Prefix + name).ToArray(); + } + public void Dispose() => _listener.Dispose(); + internal sealed record Measurement(string Name, string? Unit, double Value, KeyValuePair[] Tags) + { + public string Series => string.Join(";", Tags.Select(tag => $"{tag.Key}={tag.Value}")); + public void AssertCounter(string grainType, string? status = null) + { + Assert.Equal(1, Value); + Assert.Null(Unit); + KeyValuePair[] expected = status is null + ? [new("grain_type", grainType)] + : [new("grain_type", grainType), new("status", status)]; + Assert.Equal(expected, Tags); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs new file mode 100644 index 00000000000..ac917403d44 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs @@ -0,0 +1,147 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingRoutingAndDeadLetterTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task InboxDeadLettersRetainNewestEntriesWithinConfiguredCapacity() + { + var receiver = NewGrain(); + var messageIds = new List(); + for (var sequence = 0; sequence < 3; sequence++) + { + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 20 + sequence, + $"dead-letter-{sequence}", + ThrowDuringPreparation: true)); + messageIds.Add(envelope.Envelope.MessageId); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Any(entry => entry.MessageId == envelope.Envelope.MessageId)); + } + + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(2, state.InboxDeadLetters.Count); + Assert.DoesNotContain(state.InboxDeadLetters, entry => entry.MessageId == messageIds[0]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[1]); + Assert.Contains(state.InboxDeadLetters, entry => entry.MessageId == messageIds[2]); + } + + [Fact] + public async Task ActivationRemovesExpiredInboxDeadLetters() + { + var receiver = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(TestApplicationProtocol.NewMessageId(), + 30, + "expired-dead-letter", + ThrowDuringPreparation: true)); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var before = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Fixture.Clock.Advance(TimeSpan.FromHours(2)); + await receiver.RequestDeactivationAsync(); + var after = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId + && snapshot.InboxDeadLetters.Count == 0); + + Assert.Empty(after.InboxDeadLetters); + } + + [Fact] + public async Task MalformedTypedBody_DeadLettersAndDoesNotBlockLaterValidMessage() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + var poisoned = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + Assert.Empty(poisoned.Effects); + var deadLetter = Assert.Single(poisoned.InboxDeadLetters); + Assert.Equal(malformed.Envelope.MessageId, deadLetter.MessageId); + Assert.Contains(nameof(TestMessage), deadLetter.Reason, StringComparison.Ordinal); + + var valid = CreateEnvelope(receiver, NewMessage(41, "valid-after-poison"), "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, valid)).Status); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("valid-after-poison", Assert.Single(recovered.Effects).Value); + Assert.Single(recovered.InboxDeadLetters); + } + + [Fact] + public async Task InboxDeadLetterRemoval_IsDurable() + { + var receiver = NewGrain(); + var malformed = CreateEnvelope(receiver, "wrong-body", "typed"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, malformed)).Status); + _ = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.True(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + Assert.False(await receiver.RemoveInboxDeadLetterAsync( + malformed.Envelope.MessageId)); + + await receiver.RequestDeactivationAsync(); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); + } + + [Fact] + public async Task HandlerNotFound_IsRejectedWithoutInboxPersistence() + { + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await ReactivateAsync(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(71, "missing"), "unknown/route"); + + var result = await DeliverAsync(receiver, envelope); + + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.Equal("No inbox handler is registered.", result.Message); + var state = await receiver.GetSnapshotAsync(); + Assert.Equal(0, state.InboxCount); + Assert.Empty(state.Effects); + } + + [Fact] + public async Task Deliver_ImplicitReceiverNamespacesTheSameIncomingCommand() + { + var receiver = NewGrain(); + var declaredReceiver = NewGrain(); + var envelope = CreateEnvelope( + declaredReceiver, + NewMessage(74, "wrong-receiver")); + + Assert.Null(typeof(InboxMessage).GetProperty("ReceiverId")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + var first = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(first.Effects).LogicalId); + Assert.Equal(1, Assert.Single(first.Effects).Count); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(declaredReceiver, envelope)).Status); + var delivered = await Fixture.WaitForEffectCountAsync(declaredReceiver, 1); + Assert.Equal(envelope.Envelope.MessageId, Assert.Single(delivered.Effects).LogicalId); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(declaredReceiver, envelope)).Status); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).ProcessedMessageCount); + Assert.Equal(1, (await declaredReceiver.GetSnapshotAsync()).ProcessedMessageCount); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs new file mode 100644 index 00000000000..4b080a440a9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs @@ -0,0 +1,65 @@ +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ProcessedRetentionMaintenanceTests : MessagingBehaviorTestBase +{ + [Fact] + public async Task SnapshotWrite_PrunesExpiredProcessedIdsAndRecoveryRetainsCurrentOnes() + { + var receiver = NewGrain(); + var old = CreateEnvelope(receiver, NewMessage(120, "old")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, old)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + Fixture.Clock.Advance(TimeSpan.FromMinutes(6)); + var current = CreateEnvelope(receiver, NewMessage(121, "current")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, current)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + Fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + var fresh = CreateEnvelope(receiver, NewMessage(122, "fresh")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, fresh)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 3); + + var processed = await GetProcessedAsync(receiver); + Assert.DoesNotContain(old.Envelope.MessageId, processed); + Assert.Contains(current.Envelope.MessageId, processed); + Assert.Contains(fresh.Envelope.MessageId, processed); + + await ReactivateAsync(receiver); + var recovered = await GetProcessedAsync(receiver); + Assert.True(processed.SetEquals(recovered)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, current)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, fresh)).Status); + } + + [Fact] + public async Task ExpiredProcessedId_IsAcceptedAgainWhileUnexpiredIdsDeduplicate() + { + var receiver = NewGrain(); + var message = CreateEnvelope(receiver, NewMessage(130, "expiring")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 1); + + Fixture.Clock.Advance(TimeSpan.FromMinutes(10) - TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, message)).Status); + + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, message)).Status); + await Fixture.WaitForEffectCountAsync(receiver, 2); + } + + private Task> GetProcessedAsync(IMessagingTestGrain receiver) + { + var inbox = Fixture.GetInbox(receiver); + return Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs new file mode 100644 index 00000000000..a8bc38600e9 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/StandardStateJsonContext.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace Orleans.Messaging.Tests.Functional; + +[JsonSerializable(typeof(string))] +[JsonSerializable(typeof(int))] +[JsonSerializable(typeof(uint))] +[JsonSerializable(typeof(DateTime))] +internal partial class StandardStateJsonContext : JsonSerializerContext; diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs new file mode 100644 index 00000000000..ba4ebecc285 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -0,0 +1,144 @@ +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Hosting; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicMessagingRegistrationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AddMessaging_RepeatedPublicRegistrationRetainsOneConfiguratorAndScopedBindings(bool useSiloBuilder) + { + var builder = new TestSiloBuilder(); + builder.AddJournaling(); + var services = builder.Services; + for (var invocation = 0; invocation < 2; invocation++) + { + if (useSiloBuilder) + { + Assert.Same(builder, builder.AddMessaging()); + } + else + { + Assert.Same(services, services.AddMessaging()); + } + } + + var configurator = Assert.Single(services, descriptor => + descriptor.ServiceType == typeof(IConfigureGrainTypeComponents) + && descriptor.ImplementationType == typeof(MessagingGrainTypeConfigurator)); + Assert.Equal(ServiceLifetime.Singleton, configurator.Lifetime); + foreach (var contract in new[] + { + typeof(MessagingPump), typeof(Inbox), typeof(IInbox), typeof(IMessagingDiagnostics) + }) + { + var descriptor = Assert.Single(services, descriptor => descriptor.ServiceType == contract && !descriptor.IsKeyedService); + Assert.Equal(ServiceLifetime.Scoped, descriptor.Lifetime); + } + + // The configurator binds the inbox extension at activation; the only keyed grain extension is the durable-jobs receiver. + var extension = Assert.Single(services, descriptor => descriptor.ServiceType == typeof(IGrainExtension)); + Assert.Equal(typeof(Orleans.DurableJobs.DurableJob).Assembly, Assert.IsAssignableFrom(extension.ServiceKey).Assembly); + } + + [Fact] + public void AddMessaging_PreservesCallerSuppliedTimeProviderAndAppliesOptions() + { + var expectedTime = new FixedTimeProvider(new DateTimeOffset(2040, 2, 3, 4, 5, 6, TimeSpan.Zero)); + var services = new ServiceCollection(); + services.AddSingleton(expectedTime); + + services.AddMessaging(options => + { + options.MaxCapacity = 17; + options.InboxBatchSize = 4; + options.OutboxBatchSize = 5; + }); + using var provider = services.BuildServiceProvider(); + + Assert.Same(expectedTime, provider.GetRequiredService()); + var configured = provider.GetRequiredService>().Value; + Assert.Equal(17, configured.MaxCapacity); + Assert.Equal(4, configured.InboxBatchSize); + Assert.Equal(5, configured.OutboxBatchSize); + } + + [Theory] + [InlineData("json")] + [InlineData("orleans-binary")] + public void AddMessaging_PreservesConfiguredJournalFormat(string format) + { + var services = new ServiceCollection(); + services.Configure(options => options.JournalFormatKey = format); + services.AddMessaging(); + using var provider = services.BuildServiceProvider(); + + Assert.Equal(format, provider.GetRequiredService>().Value.JournalFormatKey); + } + + [Fact] + public void AddMessaging_InvalidOptionsFailThroughOptionsContract() + { + var services = new ServiceCollection(); + services.AddMessaging(options => options.MaxCapacity = 0); + using var provider = services.BuildServiceProvider(); + + var exception = Assert.Throws( + () => provider.GetRequiredService>().Value); + + Assert.Contains("DurableMessagingOptions validation failed", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task AddMessaging_DoesNotReplaceUnrelatedConstructionErrors() + { + var services = new ServiceCollection(); + services.AddMessaging(); + services.AddScoped(); + await using var provider = services.BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + var exception = Assert.Throws(() => scope.ServiceProvider.GetRequiredService()); + + Assert.Contains(nameof(IGrainContext), exception.Message, StringComparison.Ordinal); + } + + private sealed class FixedTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => utcNow; + } + + private sealed class TestSiloBuilder : ISiloBuilder + { + public IServiceCollection Services { get; } = new ServiceCollection(); + public IConfiguration Configuration { get; } = new ConfigurationBuilder().Build(); + } + + private sealed class ConstructionTestStateManager : IJournaledStateManager + { + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => default; + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => default; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; + public void RegisterStateMachine(string name, IStateMachine state) { } + public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? state) + { + state = null; + return false; + } + + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; + } +} diff --git a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj index 5b9a87425a4..23778326925 100644 --- a/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj +++ b/test/Orleans.Messaging.Tests/Orleans.Messaging.Tests.csproj @@ -8,11 +8,13 @@ + + diff --git a/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs new file mode 100644 index 00000000000..47e0d937673 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ActivationValidationTestGrains.cs @@ -0,0 +1,137 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ActivationValidationProbe +{ + private readonly ConcurrentDictionary> _observations = new(); + public Observation[] Get(GrainId grainId) => _observations.TryGetValue(grainId, out var values) ? values.ToArray() : []; + public Observation Track(IGrainContext context) + { + var observation = new Observation(context); + _observations.GetOrAdd(context.GrainId, static _ => new()).Enqueue(observation); + return observation; + } + + public sealed class Observation(IGrainContext context) + { + public IGrainContext Context { get; } = context; + public bool InstanceAvailableDuringConstruction { get; } = context.GrainInstance is not null; + public int Activated; + public int Calls; + } +} + +public interface IActivationValidationTestGrain : IGrainWithGuidKey +{ + Task IncrementAsync(); + Task DeactivateAsync(); +} + +public interface IAlwaysInterleaveValidationTestGrain : IActivationValidationTestGrain +{ + [AlwaysInterleave] + Task InterleaveAsync(); +} + +public abstract class ActivationValidationTestGrain : DurableGrain, IActivationValidationTestGrain +{ + private readonly IDurableValue _value; + private readonly ActivationValidationProbe.Observation _observation; + + protected ActivationValidationTestGrain(IGrainContext context, ActivationValidationProbe probe, + IDurableValue value) + { + _value = value; + _observation = probe.Track(context); + } + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _observation.Activated); + return base.OnActivateAsync(cancellationToken); + } + + public async Task IncrementAsync() + { + Interlocked.Increment(ref _observation.Calls); + _value.Value++; + await WriteStateAsync(); + return _value.Value; + } + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + +} + +public sealed class SupportedActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[Reentrant] +public sealed class ReentrantActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[StatelessWorker] +public sealed class StatelessActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +public sealed class AlwaysInterleaveActivationValidationTestGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value), IAlwaysInterleaveValidationTestGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public sealed class ExecutionPropertyAttribute(string key, string value) : Attribute, IGrainPropertiesProviderAttribute +{ + public void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) => + properties[key] = value; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "TrUe")] +public sealed class MetadataReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value) +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "false")] +public sealed class MetadataNonReentrantActivationValidationGrain( + IGrainContext context, ActivationValidationProbe probe, + [FromKeyedServices("activation-validation")] IDurableValue value) + : ActivationValidationTestGrain(context, probe, value); diff --git a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs new file mode 100644 index 00000000000..77edf5e3dd8 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs @@ -0,0 +1,151 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class DurableJobManagerProbe +{ + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _attempts = []; + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _successes = []; + private readonly ConcurrentDictionary _failures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _postScheduleFailures = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _duplicates = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary<(string JobName, GrainId Target), ConcurrentQueue> _scheduledJobs = []; + private readonly ConcurrentDictionary _scheduleBarriers = new(StringComparer.Ordinal); + + public void FailNext(string jobName) => + _failures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void FailAfterNext(string jobName) => + _postScheduleFailures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public void DuplicateNext(string jobName) => + _duplicates.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + + public int GetAttemptCount(string jobName, GrainId target) => + _attempts.TryGetValue((jobName, target), out var count) ? count : 0; + + public int GetSuccessCount(string jobName, GrainId target) => + _successes.TryGetValue((jobName, target), out var count) ? count : 0; + + public IReadOnlyList GetScheduledJobs(string jobName, GrainId target) => + _scheduledJobs.TryGetValue((jobName, target), out var jobs) ? jobs.ToArray() : []; + + public ScheduleBarrier BlockNext(string jobName) + { + var barrier = new ScheduleBarrier(); + if (!_scheduleBarriers.TryAdd(jobName, barrier)) + { + throw new InvalidOperationException($"A scheduling barrier is already armed for '{jobName}'."); + } + + return barrier; + } + + internal void OnAttempt(ScheduleJobRequest request) => + _attempts.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnSuccess(ScheduleJobRequest request) => + _successes.AddOrUpdate((request.JobName, request.Target), 1, static (_, count) => count + 1); + + internal void OnScheduled(ScheduleJobRequest request, DurableJob job) => + _scheduledJobs.GetOrAdd((request.JobName, request.Target), static _ => new()).Enqueue(job); + + internal bool ShouldFail(string jobName) + => TryConsumeFailure(_failures, jobName); + + internal bool ShouldFailAfterSchedule(string jobName) + => TryConsumeFailure(_postScheduleFailures, jobName); + + internal bool ShouldDuplicate(string jobName) + => TryConsumeFailure(_duplicates, jobName); + + internal async Task WaitIfBlockedAsync(string jobName, CancellationToken cancellationToken) + { + if (!_scheduleBarriers.TryRemove(jobName, out var barrier)) + { + return; + } + + barrier.Entered.TrySetResult(); + await barrier.Release.Task.WaitAsync(cancellationToken); + } + + private static bool TryConsumeFailure( + ConcurrentDictionary failures, + string jobName) + { + while (failures.TryGetValue(jobName, out var remaining) && remaining > 0) + { + if (failures.TryUpdate(jobName, remaining - 1, remaining)) + { + return true; + } + } + + return false; + } + + public sealed class ScheduleBarrier : IDisposable + { + internal TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public void Continue() => Release.TrySetResult(); + + public void Dispose() => Continue(); + } +} + +internal sealed class ControlledDurableJobManager( + ILocalDurableJobManager inner, + DurableJobManagerProbe probe) : ILocalDurableJobManager +{ + public async Task ScheduleJobAsync( + ScheduleJobRequest request, + CancellationToken cancellationToken) + { + probe.OnAttempt(request); + if (probe.ShouldFail(request.JobName)) + { + throw new IOException($"Injected durable job scheduling failure for '{request.JobName}'."); + } + + await probe.WaitIfBlockedAsync(request.JobName, cancellationToken); + var result = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, result); + if (probe.ShouldDuplicate(request.JobName)) + { + var duplicate = await inner.ScheduleJobAsync(request, cancellationToken); + probe.OnScheduled(request, duplicate); + } + + probe.OnSuccess(request); + if (probe.ShouldFailAfterSchedule(request.JobName)) + { + throw new IOException( + $"Injected durable job scheduling response failure for '{request.JobName}'."); + } + + return result; + } + + public Task CancelAsync(DurableJob job, CancellationToken cancellationToken) => + inner.CancelAsync(job, cancellationToken); + + public static void Decorate(IServiceCollection services, DurableJobManagerProbe probe) + { + var descriptor = services.Last(service => service.ServiceType == typeof(ILocalDurableJobManager)); + var factory = descriptor.ImplementationFactory + ?? throw new InvalidOperationException("The durable job manager registration must use an implementation factory."); + services.Remove(descriptor); + services.AddSingleton( + serviceProvider => new ControlledDurableJobManager( + (ILocalDurableJobManager)factory(serviceProvider), + probe)); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs new file mode 100644 index 00000000000..734b26bc2d2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ControlledJournalStorageProvider.cs @@ -0,0 +1,282 @@ +using System.Buffers; +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class ControlledJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog +{ + private VolatileJournalStorageProvider? _inner; + private readonly ConcurrentDictionary _readPlans = new(); + private readonly ConcurrentDictionary _deletePlans = new(); + private readonly ConcurrentDictionary _writePlans = new(); + private readonly ConcurrentDictionary _postWritePlans = new(); + private readonly ConcurrentDictionary _postDeleteFailures = new(); + private readonly ConcurrentDictionary _successfulWrites = new(); + private readonly ConcurrentDictionary _reads = new(); + private readonly ConcurrentDictionary _creations = new(); + private readonly ConcurrentDictionary _initializations = new(); + + private readonly ConcurrentDictionary _snapshots = new(); + public void RequestSnapshot(JournalId journalId) => _snapshots[journalId] = 0; + + public string? JournalFormatKey { get; private set; } + + public void Configure(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + JournalFormatKey = options.Value.JournalFormatKey; + _inner ??= new VolatileJournalStorageProvider(options); + } + + public IJournalStorage CreateStorage(JournalId journalId) + { + _creations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return new ControlledJournalStorage(this, journalId, Inner.CreateStorage(journalId)); + } + + public IAsyncEnumerable ListAsync( + JournalCatalogListOptions? options = null, + CancellationToken cancellationToken = default) => + Inner.ListAsync(options, cancellationToken); + + private VolatileJournalStorageProvider Inner => + _inner ?? throw new InvalidOperationException("The controlled journal storage provider has not been configured."); + + public WriteBarrier BlockWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + var plan = new WritePlan(matchingWrite, fail: false); + if (!_writePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public WriteBarrier BlockDelete(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_deletePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A delete plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public WriteBarrier BlockRead(JournalId journalId, int matchingRead = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingRead); + var plan = new WritePlan(matchingRead, fail: false); + if (!_readPlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A read plan is already armed for journal '{journalId}'."); + } + + return new WriteBarrier(plan); + } + + public void FailWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_writePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A write plan is already armed for journal '{journalId}'."); + } + } + + public void FailAfterWrite(JournalId journalId, int matchingWrite = 1) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(matchingWrite); + if (!_postWritePlans.TryAdd(journalId, new WritePlan(matchingWrite, fail: true))) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + } + + public WriteBarrier BlockAcknowledgement(JournalId journalId) + { + var plan = new WritePlan(1, fail: false); + if (!_postWritePlans.TryAdd(journalId, plan)) + { + throw new InvalidOperationException($"A post-write plan is already armed for journal '{journalId}'."); + } + return new WriteBarrier(plan); + } + + public void FailAfterDelete(JournalId journalId) + { + if (!_postDeleteFailures.TryAdd(journalId, 0)) + { + throw new InvalidOperationException($"A post-delete failure is already armed for journal '{journalId}'."); + } + } + + public int GetSuccessfulWriteCount(JournalId journalId) => + _successfulWrites.TryGetValue(journalId, out var count) ? count : 0; + + public int GetCreationCount(JournalId journalId) => _creations.TryGetValue(journalId, out var count) ? count : 0; + public int GetReadCount(JournalId journalId) => _reads.TryGetValue(journalId, out var count) ? count : 0; + public int GetInitializationCount(JournalId journalId) => _initializations.TryGetValue(journalId, out var count) ? count : 0; + + private async ValueTask BeforeWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_writePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _writePlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.EntryScheduler = TaskScheduler.Current; + plan.EntryContext = ReceiverTestServices.CurrentGrainContext; + plan.Entered.TrySetResult(); + if (plan.Fail) + { + throw new IOException($"Injected journal write failure for '{journalId}'."); + } + + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private async ValueTask BeforeReadAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_readPlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _readPlans.TryRemove(new KeyValuePair(journalId, plan)); + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private void OnWriteSucceeded(JournalId journalId) => + _successfulWrites.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + + private async ValueTask AfterWriteAsync(JournalId journalId, CancellationToken cancellationToken) + { + if (!_postWritePlans.TryGetValue(journalId, out var plan) + || Interlocked.Increment(ref plan.Seen) != plan.Target) + { + return; + } + + _postWritePlans.TryRemove(new KeyValuePair(journalId, plan)); + if (plan.Fail) + { + throw new IOException($"Injected post-commit journal response failure for '{journalId}'."); + } + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + internal sealed class WritePlan(int target, bool fail) + { + public int Target { get; } = target; + public bool Fail { get; } = fail; + public int Seen; + public TaskScheduler? EntryScheduler; + public IGrainContext? EntryContext; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public sealed class WriteBarrier : IDisposable + { + private readonly WritePlan _plan; + + internal WriteBarrier(WritePlan plan) => _plan = plan; + + public TaskScheduler? EntryScheduler => _plan.EntryScheduler; + public IGrainContext? EntryContext => _plan.EntryContext; + public Task WaitUntilEnteredAsync() => _plan.Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => _plan.Release.TrySetResult(); + public void Dispose() => Release(); + public void Fail() => Fail(new IOException("Injected blocked journal write failure.")); + public void Fail(Exception exception) => _plan.Release.TrySetException(exception); + } + + private sealed class ControlledJournalStorage( + ControlledJournalStorageProvider owner, + JournalId journalId, + IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => owner._snapshots.ContainsKey(journalId) || inner.IsCompactionRequested; + + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + var grain = ReceiverTestServices.CurrentGrainContext?.GrainInstance as MessagingTestGrain; + owner._reads.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + await owner.BeforeReadAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReadAsync(consumer, cancellationToken).ConfigureAwait(false); + grain?.CaptureStorageRead(); + } + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) + { + owner._initializations.AddOrUpdate(journalId, 1, static (_, count) => count + 1); + return inner.CreateIfNotExistsAsync(metadata, cancellationToken); + } + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var named = instance as NamedFactoryMessagingGrain; + var captured = grain?.CaptureStorageWrite(); + var namedSnapshot = named?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + owner._snapshots.TryRemove(journalId, out _); + owner.OnWriteSucceeded(journalId); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + if (namedSnapshot is not null) named!.PublishStoredSnapshot(namedSnapshot); + } + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var instance = ReceiverTestServices.CurrentGrainContext?.GrainInstance; + var grain = instance as MessagingTestGrain; + var named = instance as NamedFactoryMessagingGrain; + var captured = grain?.CaptureStorageWrite(); + var namedSnapshot = named?.CaptureStorageWrite(); + await owner.BeforeWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + await inner.AppendAsync(value, cancellationToken).ConfigureAwait(false); + owner.OnWriteSucceeded(journalId); + await owner.AfterWriteAsync(journalId, cancellationToken).ConfigureAwait(false); + if (captured is not null) grain!.PublishStoredSnapshot(captured); + if (namedSnapshot is not null) named!.PublishStoredSnapshot(namedSnapshot); + } + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + if (owner._deletePlans.TryRemove(journalId, out var plan)) + { + plan.Entered.TrySetResult(); + await plan.Release.Task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + await inner.DeleteAsync(cancellationToken).ConfigureAwait(false); + if (owner._postDeleteFailures.TryRemove(journalId, out _)) + { + throw new IOException($"Injected post-delete journal response failure for '{journalId}'."); + } + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs new file mode 100644 index 00000000000..02441689ec6 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/FaultingInboxCodecFixture.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class FaultingInboxCodecFixture : MessagingClusterFixture +{ + public Exception? NextFailure { get; set; } + public bool FailOnSnapshot { get; set; } + public bool? FailedSnapshot { get; private set; } + protected override void ConfigureServices(IServiceCollection services) + { + var descriptor = services.Last(entry => entry.ServiceType == typeof(IDurableValueCommandCodec<>) + && Equals(entry.ServiceKey, "orleans-binary")); + var implementation = descriptor.KeyedImplementationType!.MakeGenericType(typeof(InboxCommand)); + services.AddKeyedSingleton>("orleans-binary", (sp, _) => + new FaultingCodec(this, (IDurableValueCommandCodec)ActivatorUtilities.CreateInstance(sp, implementation))); + } + + private sealed class FaultingCodec(FaultingInboxCodecFixture owner, IDurableValueCommandCodec inner) + : IDurableValueCommandCodec + { + public void WriteSet(InboxCommand value, JournalStreamWriter writer) + { + var snapshot = value.Kind == InboxCommandKind.Snapshot; + if (snapshot == owner.FailOnSnapshot && owner.NextFailure is { } failure) + { + owner.NextFailure = null; + owner.FailedSnapshot = snapshot; + throw failure; + } + + inner.WriteSet(value, writer); + } + + public void Apply(JournalBufferReader input, IDurableValueCommandHandler consumer) => inner.Apply(input, consumer); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs new file mode 100644 index 00000000000..0ecb2480c55 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs @@ -0,0 +1,53 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class HandlerProbe +{ + private readonly ConcurrentDictionary<(GrainId GrainId, string Route), Barrier> _barriers = new(); + + public Barrier Arm(GrainId grainId, string route) + { + var barrier = new Barrier(this, (grainId, route)); + if (!_barriers.TryAdd((grainId, route), barrier)) + { + throw new InvalidOperationException($"A handler barrier is already armed for '{grainId}' and route '{route}'."); + } + + return barrier; + } + + public bool TryGet( + GrainId grainId, + string route, + [NotNullWhen(true)] out Barrier? barrier) => + _barriers.TryGetValue((grainId, route), out barrier); + + public sealed class Barrier : IDisposable + { + private readonly HandlerProbe _owner; + private readonly (GrainId GrainId, string Route) _key; + + internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key) + { + _owner = owner; + _key = key; + } + + internal TaskCompletionSource Entered { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + internal TaskCompletionSource Continue { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void Release() => Continue.TrySetResult(); + + public void Dispose() + { + Release(); + _owner._barriers.TryRemove(new KeyValuePair<(GrainId GrainId, string Route), Barrier>(_key, this)); + } + } +} diff --git a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs new file mode 100644 index 00000000000..0294304121d --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs @@ -0,0 +1,44 @@ +using System.Runtime.CompilerServices; + +namespace Orleans.Messaging.Tests.Support; + +internal static class InboxTestAccess +{ + + public static IEnumerable Messages(this Inbox inbox) => inbox.MessageState.Values.Select(static entry => entry.Message); + + public static int Count(this IOutbox outbox) => outbox switch + { + JournaledTestOutbox value => value.Count, + _ => throw new NotSupportedException(outbox.GetType().FullName) + }; + + public static IEnumerable Messages(this IOutbox outbox) => outbox switch + { + JournaledTestOutbox value => value.Messages, + _ => throw new NotSupportedException(outbox.GetType().FullName) + }; + + public static int GetProcessedCount(Inbox inbox) => GetProcessed(inbox).Count; + + public static HashSet GetProcessedKeys(Inbox inbox) => [.. GetProcessed(inbox).Keys]; + + public static bool TryGetProcessedAt(Inbox inbox, HierarchicalKey messageId, out DateTimeOffset processedAt) => + GetProcessed(inbox).TryGetValue(messageId, out processedAt); + + public static void SetProcessedAt(Inbox inbox, HierarchicalKey messageId, DateTimeOffset processedAt) => + GetProcessed(inbox)[messageId] = processedAt; + + public static IReadOnlyDictionary GetPendingMessages(Inbox inbox) => inbox.MessageState; + + public static IReadOnlyDictionary GetProcessedMessages(Inbox inbox) => GetProcessed(inbox); + + public static InboxMessageState GetMessageState(Inbox inbox, HierarchicalKey messageId) => + inbox.MessageState[messageId].State; + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_messages")] + private static extern ref Dictionary GetMessages(Inbox inbox); + + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_processed")] + private static extern ref Dictionary GetProcessed(Inbox inbox); +} diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs new file mode 100644 index 00000000000..6109b7c0598 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -0,0 +1,63 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +// Captures opaque handler output in the same journal as inbox effects. Dispatch belongs to the outbox layer. +internal sealed class JournaledTestOutbox(IDurableDictionary messages) + : IOutbox, ILifecycleObserver, IEnumerable> +{ + private bool _stopped; + private readonly TaskCompletionSource _stopping = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task Stopping => _stopping.Task; + public int SendCalls { get; private set; } + public Exception? NextSendFailure { get; set; } + public IDurableDictionary StoredMessages { get; } = messages; + public int Count => StoredMessages.Count; + public IEnumerable Messages => StoredMessages.Values; + public IEnumerator> GetEnumerator() => StoredMessages.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public void Send(OutboxMessage envelope) + { + SendCalls++; + if (_stopped) throw new InvalidOperationException("The test outbox owner is stopped."); + if (envelope.Envelope.MessageId.IsDefault || envelope.ReceiverId.IsDefault + || envelope.Envelope.Payload is null) + { + throw new ArgumentException("An outgoing envelope requires identities and opaque payload bytes.", nameof(envelope)); + } + if (TryGetMessage(envelope.Envelope.MessageId, out var existing)) + { + if (existing.ReceiverId != envelope.ReceiverId + || !string.Equals(TestApplicationProtocol.Subject(existing.Envelope), + TestApplicationProtocol.Subject(envelope.Envelope), StringComparison.Ordinal) + || !existing.Envelope.Payload.AsSpan().SequenceEqual(envelope.Envelope.Payload)) + { + throw new InvalidOperationException($"The outbox already contains a different envelope with message ID '{envelope.Envelope.MessageId}'."); + } + return; + } + if (NextSendFailure is { } failure) + { + NextSendFailure = null; + throw failure; + } + StoredMessages.Add(envelope.Envelope.MessageId, envelope); + } + + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => + StoredMessages.TryGetValue(messageId, out envelope); + public Task OnStart(CancellationToken cancellationToken) => Task.CompletedTask; + public Task OnStop(CancellationToken cancellationToken) + { + _stopped = true; + _stopping.TrySetResult(); + return Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs new file mode 100644 index 00000000000..ba6a14e97e3 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs @@ -0,0 +1,83 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public abstract class MessagingBehaviorTestBase : IAsyncLifetime +{ + protected MessagingBehaviorTestBase(bool receiverOnly = true) + : this(new MessagingClusterFixture(receiverOnly)) + { + } + + protected MessagingBehaviorTestBase(MessagingClusterFixture fixture) => Fixture = fixture; + + protected MessagingClusterFixture Fixture { get; } + protected SerializerSessionPool Sessions => Fixture.Client.ServiceProvider.GetRequiredService(); + + public ValueTask InitializeAsync() => Fixture.InitializeAsync(); + public ValueTask DisposeAsync() => Fixture.DisposeAsync(); + + protected IMessagingTestGrain NewGrain() => + Fixture.Client.GetGrain(Guid.NewGuid()); + + protected static TestMessage NewMessage(int sequence, string value) => + new(TestApplicationProtocol.NewMessageId(), sequence, value); + + protected async Task ReactivateAsync(IMessagingTestGrain receiver) + { + var previous = Fixture.GetGrainContext(receiver); + await receiver.RequestDeactivationAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + Assert.NotSame(previous, Fixture.GetGrainContext(receiver)); + } + + protected static Task DeliverAsync( + IMessagingTestGrain receiver, + InboxMessage envelope) => + DeliverWithCancellationAsync(receiver, envelope, TestContext.Current.CancellationToken); + + protected static async Task DeliverWithCancellationAsync( + IMessagingTestGrain receiver, + InboxMessage envelope, + CancellationToken cancellationToken) => + await receiver.AsReference().DeliverAsync(envelope, cancellationToken); + + protected static async Task WaitForBarrierAsync( + IMessagingTestGrain receiver, + HandlerProbe.Barrier barrier) + { + try + { + await barrier.WaitUntilEnteredAsync(); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Handler did not start. Inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, maxHandlers={snapshot.MaxConcurrentHandlers}, deadLetters={string.Join(" | ", snapshot.InboxDeadLetters.Select(static item => item.Reason))}.", + exception); + } + } + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + TestMessage message, + string route = "messages/record") => + CreateEnvelope(receiver, (object)message, route); + + protected InboxMessage CreateEnvelope( + IMessagingTestGrain receiver, + object body, + string route) + { + var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); + var sender = GrainId.Create("external-test-sender", Guid.NewGuid().ToString("N")); + return TestApplicationProtocol.CreateInbox(sessions, sender, route, body); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs new file mode 100644 index 00000000000..cd97dd92ee6 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -0,0 +1,272 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public class MessagingClusterFixture : IAsyncLifetime +{ + public MessagingClusterFixture() : this(receiverOnly: true) + { + } + + public MessagingClusterFixture(bool receiverOnly) : this(1, receiverOnly) + { + } + + protected MessagingClusterFixture(int initialSilos, bool receiverOnly = false, bool useServiceCollection = false) + { + Clock = new FakeTimeProvider(DateTimeOffset.UtcNow); + Storage = new ControlledJournalStorageProvider(); + Metrics = new MessagingMetricProbe(); + JobManagerProbe = new DurableJobManagerProbe(); + HandlerProbe = new HandlerProbe(); + SnapshotProbe = new SnapshotProbe(); + ActivationProbe = new ActivationValidationProbe(); + var clusterId = $"messaging-{Guid.NewGuid():N}"; + var serviceId = $"messaging-service-{Guid.NewGuid():N}"; + var builder = new InProcessTestClusterBuilder((short)initialSilos); + builder.ConfigureClient(clientBuilder => + clientBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + })); + builder.ConfigureSilo((_, siloBuilder) => + { + siloBuilder.Configure(options => + { + options.ClusterId = clusterId; + options.ServiceId = serviceId; + }); + siloBuilder.Services.AddSingleton(Clock); + siloBuilder.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + siloBuilder.Services.AddSingleton(HandlerProbe); + siloBuilder.Services.AddSingleton(new TestHandlerConfiguration()); + siloBuilder.Services.AddSingleton(SnapshotProbe); + siloBuilder.Services.AddSingleton(ActivationProbe); + siloBuilder.UseInMemoryDurableJobs(); + // The test grains journal Orleans-serialized types. + siloBuilder.Services.Configure(options => options.JournalFormatKey = "orleans-binary"); + if (receiverOnly) + { + ReceiverTestServices.Add(siloBuilder.Services, ConfigureOptions); + } + else if (useServiceCollection) + { + siloBuilder.Services.AddMessaging(ConfigureOptions); + } + else + { + siloBuilder.AddMessaging(ConfigureOptions); + } + siloBuilder.ConfigureServices(services => + ControlledDurableJobManager.Decorate(services, JobManagerProbe)); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.RemoveAll(); + siloBuilder.Services.AddSingleton(Storage); + siloBuilder.Services.AddSingleton(serviceProvider => + { + Storage.Configure(serviceProvider.GetRequiredService>()); + return Storage; + }); + siloBuilder.Services.AddSingleton( + serviceProvider => (IJournalStorageCatalog)serviceProvider.GetRequiredService()); + ConfigureServices(siloBuilder.Services); + }); + Cluster = builder.Build(); + } + + public InProcessTestCluster Cluster { get; } + public IClusterClient Client => Cluster.Client!; + public FakeTimeProvider Clock { get; } + public ControlledJournalStorageProvider Storage { get; } + public MessagingMetricProbe Metrics { get; } + public DurableJobManagerProbe JobManagerProbe { get; } + public HandlerProbe HandlerProbe { get; } + public SnapshotProbe SnapshotProbe { get; } + public ActivationValidationProbe ActivationProbe { get; } + + public Task WaitForEffectCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.Effects.Sum(static effect => effect.Count) >= expected); + + public Task WaitForInboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.InboxCount == expected); + + public Task WaitForDeadLetterCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync( + grain.GetGrainId(), + snapshot => snapshot.InboxDeadLetters.Count + snapshot.OutboxDeadLetters.Count >= expected); + + public Task WaitForOutboxCountAsync(IMessagingTestGrain grain, int expected) => + SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.OutboxCount == expected); + + public EndpointSnapshot GetSnapshot(IMessagingTestGrain grain) => + GetGrainInstance(grain).GetSnapshotForTest(); + + public OutboxMessage[] GetStagedOutput(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService().Messages().ToArray(); + + public ValueTask WriteStateAsync(IMessagingTestGrain grain) => + GetGrainContext(grain).ActivationServices + .GetRequiredService() + .WriteStateAsync(TestContext.Current.CancellationToken); + + internal Inbox GetInbox(IAddressable grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService(); + + internal MessagingPump GetPump(IAddressable grain) => + GetGrainContext(grain).ActivationServices.GetRequiredService(); + + /// + /// Schedules an additional pump wake-up. Wake-ups are at-least-once hints, so extra ones must be harmless. + /// + public async Task WakeAsync(IAddressable grain) + { + var services = Cluster.TryGetGrainContext(grain.GetGrainId(), out var context) + ? context.ActivationServices + : Cluster.Silos[0].ServiceProvider; + await services.GetRequiredService().ScheduleJobAsync( + new ScheduleJobRequest + { + Target = grain.GetGrainId(), + JobName = MessagingPump.JobName, + DueTime = DateTimeOffset.UtcNow + }, + TestContext.Current.CancellationToken); + } + + /// Executes the pump's durable job callback directly, as a spurious or duplicate wake-up would. + internal async Task RunPumpAsync(IAddressable grain) + { + var job = new DurableJob + { + Id = Guid.NewGuid().ToString("N"), + Name = MessagingPump.JobName, + ShardId = "test", + TargetGrainId = grain.GetGrainId(), + DueTime = DateTimeOffset.UtcNow + }; + var pump = GetPump(grain); + var run = await OnGrainAsync(grain, () => pump.ExecuteJobAsync( + new PumpRunContext(job), + TestContext.Current.CancellationToken).AsTask()); + return await run.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + /// Runs on the grain's activation scheduler. + internal Task OnGrainAsync(IAddressable grain, Func func) + { + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + GetGrainContext(grain).Scheduler.QueueAction(() => + { + try + { + done.SetResult(func()); + } + catch (Exception exception) + { + done.SetException(exception); + } + }); + return done.Task; + } + + internal Task OnGrainAsync(IAddressable grain, Action action) => + OnGrainAsync(grain, () => + { + action(); + return true; + }); + + /// Polls on the grain's scheduler until it holds. + internal async Task WaitUntilAsync(IAddressable grain, Func predicate) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!await OnGrainAsync(grain, predicate)) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Condition was not met for grain '{grain.GetGrainId()}'."); + } + + await Task.Delay(10, TestContext.Current.CancellationToken); + } + } + + internal IGrainContext GetGrainContext(IAddressable grain) + { + if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) + { + throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' is not active."); + } + + return context; + } + + private MessagingTestGrain GetGrainInstance(IMessagingTestGrain grain) => + GetGrainContext(grain).GrainInstance as MessagingTestGrain + ?? throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' has an unexpected implementation."); + + private sealed class PumpRunContext(DurableJob job) : IJobRunContext + { + public DurableJob Job { get; } = job; + public string RunId { get; } = Guid.NewGuid().ToString("N"); + public int DequeueCount => 1; + } + + protected virtual void ConfigureServices(IServiceCollection services) { } + + protected virtual void ConfigureOptions(DurableMessagingOptions options) + { + options.MaxCapacity = 2; + options.DeduplicationWindow = TimeSpan.FromMinutes(10); + options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); + options.MaxProcessingAttempts = 1; + options.MaxDeliveryAttempts = 3; + options.MaxRetainedDeadLetters = 2; + options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(25); + options.InboxBatchSize = 8; + options.OutboxBatchSize = 8; + } + + public ValueTask InitializeAsync() => new(Cluster.DeployAsync()); + public async ValueTask DisposeAsync() + { + await Cluster.DisposeAsync(); + Metrics.Dispose(); + } +} + +public sealed class MultiSiloMessagingClusterFixture : MessagingClusterFixture +{ + public MultiSiloMessagingClusterFixture() : base(2) + { + } +} + +public sealed class DedupeExpiryClusterFixture : MessagingClusterFixture +{ +} + +public sealed class InboxCapacityClusterFixture : MessagingClusterFixture +{ + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 1; + options.MaxProcessingAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromHours(1); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs new file mode 100644 index 00000000000..c07cf9cc4a4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs @@ -0,0 +1,121 @@ +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class MessagingMetricProbe : IDisposable +{ + private readonly ConcurrentDictionary<(string Instrument, string TagValue), long> _measurements = []; + private readonly ConcurrentDictionary _gauges = []; + private readonly object _lock = new(); + private TaskCompletionSource _changed = CreateSignal(); + private readonly MeterListener _listener; + + public MessagingMetricProbe() + { + _listener = new MeterListener + { + InstrumentPublished = static (instrument, listener) => + { + if (instrument.Meter.Name == "Microsoft.Orleans" + && instrument.Name is "orleans-messaging-orphaned-jobs-reclaimed" + or "orleans-messaging-inbox-messages-processed" + or "orleans-durablejobs-job-attempts-started" + or "orleans-durablejobs-handler-executions-started" + or "orleans-durablejobs-jobs-completed" + or "orleans-durablejobs-jobs-rescheduled" + or "orleans-messaging-inbox-depth" + or "orleans-messaging-outbox-depth") + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + _listener.SetMeasurementEventCallback(OnMeasurement); + _listener.Start(); + } + + public long GetCount(string instrument, string tagValue = "") => + _measurements.TryGetValue((instrument, tagValue), out var count) ? count : 0; + + public long GetDepth(string instrument) + { + _listener.RecordObservableInstruments(); + return _gauges.TryGetValue(instrument, out var value) ? value : 0; + } + + public Task WaitForCountAsync( + string instrument, + long expected, + string tagValue = "") => + WaitForCountWithCancellationAsync( + instrument, + expected, + tagValue, + TestContext.Current.CancellationToken); + + public async Task WaitForCountWithCancellationAsync( + string instrument, + long expected, + string tagValue, + CancellationToken cancellationToken) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + while (GetCount(instrument, tagValue) < expected) + { + Task changed; + lock (_lock) + { + if (GetCount(instrument, tagValue) >= expected) + { + return; + } + + changed = _changed.Task; + } + + await changed.WaitAsync(timeout.Token); + } + } + + public void Dispose() => _listener.Dispose(); + + private void OnMeasurement( + Instrument instrument, + long measurement, + ReadOnlySpan> tags, + object? state) + { + if (instrument.Name is "orleans-messaging-inbox-depth" or "orleans-messaging-outbox-depth") + { + _gauges[instrument.Name] = measurement; + return; + } + + var tagValue = ""; + var tagName = instrument.Name == "orleans-messaging-inbox-messages-processed" ? "status" : "job_name"; + foreach (var tag in tags) + { + if (tag.Key == tagName) + { + tagValue = tag.Value as string ?? ""; + break; + } + } + + _measurements.AddOrUpdate( + (instrument.Name, tagValue), + measurement, + (_, current) => current + measurement); + lock (_lock) + { + _changed.TrySetResult(); + _changed = CreateSignal(); + } + } + + private static TaskCompletionSource CreateSignal() => + new(TaskCreationOptions.RunContinuationsAsynchronously); +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs new file mode 100644 index 00000000000..65c8bc46837 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -0,0 +1,469 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Concurrency; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging; +using Orleans.Runtime; +using Orleans.Runtime.Diagnostics; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +public interface IMessagingTestGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId target, string route, TestMessage message); + Task SendDuplicateAsync(GrainId target, string route, TestMessage message); + Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message); + Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message); + Task RetryWriteStateAsync(); + Task StageEffectAsync(MessageEffect effect); + Task StageOutputAsync(OutboxMessage envelope); + Task AcceptAndDeactivateAsync(InboxMessage envelope); + Task ConfigureHandlerAsync(bool enabled); + Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); + Task SetControlEnvelopeAsync(InboxMessage envelope); + Task DeleteStateAndDeactivateAsync(); + Task HoldPumpTurnAsync(string barrierRoute, bool deactivate); + Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate); +} + +[GenerateSerializer, Immutable] +public sealed record TestMessage( + [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(1)] int Sequence, + [property: Id(2)] string Value, + [property: Id(3)] GrainId? ForwardTo = null, + [property: Id(8)] bool ThrowDuringPreparation = false, + [property: Id(5)] bool CommitDuringHandling = false, + [property: Id(6)] bool DeleteDuringHandling = false, + [property: Id(9)] bool ThrowOnceDuringPreparation = false); + +[GenerateSerializer, Immutable] +public sealed record MessageEffect( + [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(1)] int Count, + [property: Id(2)] int Sequence, + [property: Id(3)] string Value); + +[GenerateSerializer, Immutable] +public sealed record EndpointSnapshot( + [property: Id(0)] Guid ActivationId, + [property: Id(1)] string SiloAddress, + [property: Id(2)] int InboxCount, + [property: Id(3)] int OutboxCount, + [property: Id(4)] int MaxConcurrentHandlers, + [property: Id(5)] IReadOnlyList Effects, + [property: Id(6)] IReadOnlyList InboxDeadLetters, + [property: Id(7)] IReadOnlyList OutboxDeadLetters, + [property: Id(9)] int ProcessedMessageCount); + +[GenerateSerializer, Immutable] +public sealed record DeadLetterSnapshot( + [property: Id(0)] HierarchicalKey MessageId, + [property: Id(1)] string Route, + [property: Id(2)] string Reason, + [property: Id(3)] int AttemptCount, + [property: Id(4)] DateTimeOffset DeadLetteredAt); + +[GrainType("messaging-inbox-test")] +public sealed class MessagingTestGrain : DurableGrain, IMessagingTestGrain, IDurableJobHandler, + IObserver, IDisposable +{ + private readonly IGrainContext _grainContext; + private readonly IDisposable _lifecycleSubscription; + private readonly IJournaledStateManager _journalOwner; + private readonly IOutbox _outbox; + private readonly IMessagingDiagnostics _diagnostics; + private readonly IDurableDictionary _effects; + private readonly Inbox _inboxState; + private readonly IInbox _inbox; + private readonly SerializerSessionPool _sessions; + private readonly TestHandlerConfiguration _handlerConfiguration; + internal IInboxHandler? HandlerOverride { get; set; } + internal Exception? BeforeJournalOperationFailure { get; set; } + internal Exception? AfterJournalOperationFailure { get; set; } + private readonly ILocalSiloDetails _siloDetails; + private readonly HandlerProbe _handlerProbe; + private readonly SnapshotProbe _snapshotProbe; + private readonly Guid _activationId = Guid.NewGuid(); + private int _activeHandlers; + private int _maxConcurrentHandlers; + private readonly HashSet _failedOnce = []; + + public MessagingTestGrain( + IGrainContext grainContext, + IJournaledStateManager journalOwner, + IInbox inbox, + IOutbox outbox, + IMessagingDiagnostics diagnostics, + [FromKeyedServices("test-effects")] IDurableDictionary effects, + [FromKeyedServices("inbox")] IDurableValue applicationInboxState, + + + SerializerSessionPool sessions, + TestHandlerConfiguration handlerConfiguration, + ILocalSiloDetails siloDetails, + HandlerProbe handlerProbe, + SnapshotProbe snapshotProbe) + { + _grainContext = grainContext; + _journalOwner = journalOwner; + _inbox = inbox; + _outbox = outbox; + _diagnostics = diagnostics; + _effects = effects; + ArgumentNullException.ThrowIfNull(applicationInboxState); + + _inboxState = grainContext.ActivationServices.GetRequiredService(); + + _sessions = sessions; + _handlerConfiguration = handlerConfiguration; + _siloDetails = siloDetails; + _handlerProbe = handlerProbe; + _snapshotProbe = snapshotProbe; + _lifecycleSubscription = GrainLifecycleEvents.AllEvents.Subscribe(this); + _journalOwner.AddHookAsync(new ControlledJournalHook(this)).GetAwaiter().GetResult(); + } + + public override async Task OnActivateAsync(CancellationToken cancellationToken) + { + if (_handlerConfiguration.IsEnabled(this.GetGrainId())) + { + _inbox.Register(new ApplicationDispatcher(this)); + } + await base.OnActivateAsync(cancellationToken); + _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + } + + public async Task SendAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendDuplicateAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + _outbox.Send(envelope); + await WriteStateAsync(); + return envelope.Envelope.MessageId; + } + + public async Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message) + { + var messageId = await SendAsync(target, route, message); + DeactivateOnIdle(); + return messageId; + } + + public Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message) + { + var envelope = CreateEnvelope(target, route, message); + _outbox.Send(envelope); + return Task.FromResult(envelope.Envelope.MessageId); + } + + public async Task RetryWriteStateAsync() => await WriteStateAsync(); + + public async Task AcceptAndDeactivateAsync(InboxMessage envelope) + { + var extension = _inboxState; + var result = await extension.DeliverAsync(envelope); + AcceptedSnapshot = CreateSnapshot(); + DeactivateOnIdle(); + return result; + } + + public Task StageOutputAsync(OutboxMessage envelope) + { + _outbox.Send(envelope); + return Task.CompletedTask; + } + + public Task StageEffectAsync(MessageEffect effect) + { + _effects[effect.LogicalId] = effect; + return Task.CompletedTask; + } + + public Task ConfigureHandlerAsync(bool enabled) + { + _handlerConfiguration.Set(this.GetGrainId(), enabled); + return Task.CompletedTask; + } + + public async Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId) + { + if (!_diagnostics.RemoveInboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); + + internal EndpointSnapshot GetSnapshotForTest() => CreateSnapshot(); + + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + public async Task DeleteStateAndDeactivateAsync() + { + try + { + await ((ILifecycleObserver)ServiceProvider.GetRequiredService()).OnStop(CancellationToken.None); + if (_outbox is ILifecycleObserver testOutbox) + { + await testOutbox.OnStop(CancellationToken.None); + } + + await _journalOwner.DeleteStateAsync(CancellationToken.None); + } + finally + { + DeactivateOnIdle(); + } + } + + internal EndpointSnapshot? AcceptedSnapshot { get; private set; } + + private InboxMessage? _controlEnvelope; + internal TaskCompletionSource ControlDeliveryEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task SetControlEnvelopeAsync(InboxMessage envelope) + { + _controlEnvelope = envelope; + return Task.CompletedTask; + } + + internal TaskScheduler? JobScheduler { get; private set; } + internal IGrainContext? JobGrainContext { get; private set; } + + public async Task ExecuteJobAsync(IJobRunContext context, CancellationToken attemptCancellationToken) + { + JobScheduler = TaskScheduler.Current; + JobGrainContext = ReceiverTestServices.CurrentGrainContext; + switch (context.Job.Name) + { + case "test/deliver-envelope": + ControlDeliveryEntered.TrySetResult(); + var extension = _inboxState; + await extension.DeliverAsync(_controlEnvelope ?? throw new InvalidOperationException("A control envelope must be configured."), attemptCancellationToken); + break; + case "test/write-journal": + await StateManager.WriteStateAsync(attemptCancellationToken); + break; + case "test/probe-scheduler": + break; + default: + throw new NotSupportedException($"Unknown test job '{context.Job.Name}'."); + } + } + + public Task HoldPumpTurnAsync(string barrierRoute, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, null, deactivate); + + public Task HoldPumpTurnAsync(string barrierRoute, InboxMessage replacement, bool deactivate) => + HoldPumpTurnCoreAsync(barrierRoute, replacement, deactivate); + + private async Task HoldPumpTurnCoreAsync(string barrierRoute, InboxMessage? replacement, bool deactivate) + { + if (!_handlerProbe.TryGet(this.GetGrainId(), barrierRoute, out var barrier)) + { + throw new InvalidOperationException("The pump-turn barrier must be armed."); + } + barrier.Entered.TrySetResult(); + await barrier.Continue.Task; + if (replacement is { } envelope) + { + var extension = _inboxState; + await extension.DeliverAsync(envelope); + } + if (deactivate) + { + DeactivateOnIdle(); + } + } + + internal TaskCompletionSource DeactivationFailure { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + void IObserver.OnNext(GrainLifecycleEvents.LifecycleEvent value) + { + if (value is GrainLifecycleEvents.Deactivating { Reason.Exception: { } exception } deactivating + && ReferenceEquals(deactivating.GrainContext, _grainContext)) + { + DeactivationFailure.TrySetResult(exception); + } + } + + void IObserver.OnError(Exception error) => DeactivationFailure.TrySetException(error); + void IObserver.OnCompleted() { } + public void Dispose() + { + _controlEnvelope = null; + _lifecycleSubscription.Dispose(); + } + + private readonly ConcurrentQueue _captures = new(); + private readonly ConcurrentQueue _outputCaptures = new(); + internal IReadOnlyList Captures => _captures.ToArray(); + internal IReadOnlyList OutputCaptures => _outputCaptures.ToArray(); + internal Exception? NextApplyFailure { get; set; } + internal TaskCompletionSource ApplyAttempted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal void ClearCaptures() => _captures.Clear(); + + internal EndpointSnapshot CaptureStorageWrite() + { + var snapshot = CreateSnapshot(); + _captures.Enqueue(snapshot); + _outputCaptures.Enqueue(_outbox.Messages().Select(static envelope => envelope.Envelope.MessageId).ToArray()); + return snapshot; + } + + internal void PublishStoredSnapshot(EndpointSnapshot snapshot) => _snapshotProbe.Publish(this.GetGrainId(), snapshot); + internal void CaptureStorageRead() => ReplayedSnapshot = CreateSnapshot(); + internal EndpointSnapshot? ReplayedSnapshot { get; private set; } + + private OutboxMessage CreateEnvelope(GrainId target, string route, TestMessage message) => + TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, route, message); + + private async ValueTask HandleAsync( + TestMessage message, + IInboxHandlerContext context, + CancellationToken cancellationToken) + { + var active = Interlocked.Increment(ref _activeHandlers); + _maxConcurrentHandlers = Math.Max(_maxConcurrentHandlers, active); + try + { + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route, out var gate)) + { + gate.Entered.TrySetResult(); + await gate.Continue.Task.WaitAsync(cancellationToken); + } + + if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route + "/application-preparation", out var preparation)) + { + preparation.Entered.TrySetResult(); + await preparation.Continue.Task.WaitAsync(cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + if (message.CommitDuringHandling) + { + await WriteStateAsync(cancellationToken); + } + if (message.ThrowDuringPreparation || (message.ThrowOnceDuringPreparation && _failedOnce.Add(message.LogicalId))) + { + throw new InvalidOperationException($"Injected handler preparation failure for {message.LogicalId}."); + } + + var outgoing = message.ForwardTo is { } destination + ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), destination, "messages/forwarded", + message with { ForwardTo = null, ThrowDuringPreparation = false }, + context.Message.Envelope.MessageId.CreateChildKey("forwarded")) + : (OutboxMessage?)null; + if (NextApplyFailure is { } failure) + { + NextApplyFailure = null; + throw failure; + } + _effects.TryGetValue(message.LogicalId, out var prior); + _effects[message.LogicalId] = new MessageEffect(message.LogicalId, (prior?.Count ?? 0) + 1, message.Sequence, message.Value); + ApplyAttempted.TrySetResult(); + if (outgoing is { } output) + { + _outbox.Send(output); + if (TestApplicationProtocol.Read(_sessions, context.Message).Route == "messages/duplicate-output") + { + _outbox.Send(output); + } + } + context.Complete(); + } + finally + { + Interlocked.Decrement(ref _activeHandlers); + } + } + + private void PublishSnapshot() => _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); + + private EndpointSnapshot CreateSnapshot() => + new( + _activationId, + _siloDetails.SiloAddress.ToParsableString(), + _inboxState.Count, + _outbox.Count(), + _maxConcurrentHandlers, + _effects.Values.OrderBy(static effect => effect.Sequence).ToArray(), + _diagnostics.InboxDeadLetters.Select(ToSnapshot).ToArray(), + [], + InboxTestAccess.GetProcessedCount(_inboxState)); + + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + + private sealed class ApplicationDispatcher(MessagingTestGrain owner) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) => true; + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (owner.HandlerOverride is { } handler) + { + return handler.HandleAsync(context, cancellationToken); + } + + var application = TestApplicationProtocol.Read(owner._sessions, context.Message); + if (!application.Route.StartsWith("messages/", StringComparison.Ordinal) && application.Route != "typed") + { + throw new InvalidOperationException($"Unknown application route '{application.Route}'."); + } + return owner.HandleAsync(application.Body as TestMessage + ?? throw new InvalidOperationException($"Expected {nameof(TestMessage)} application payload."), + context, cancellationToken); + } + } + + private sealed class ControlledJournalHook(MessagingTestGrain owner) : IJournaledStateHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (owner.BeforeJournalOperationFailure is { } failure) throw failure; + return default; + } + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + if (owner.AfterJournalOperationFailure is { } failure) throw failure; + return default; + } + } +} + +// Tests can change registration between real activations without adding a production registry API. +public sealed class TestHandlerConfiguration +{ + private readonly ConcurrentDictionary _enabled = new(); + public bool IsEnabled(GrainId id) => !_enabled.TryGetValue(id, out var enabled) || enabled; + public void Set(GrainId id, bool enabled) => _enabled[id] = enabled; +} diff --git a/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs new file mode 100644 index 00000000000..b189eb93170 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs @@ -0,0 +1,111 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +public interface INamedFactoryMessagingGrain : IGrainWithGuidKey +{ + Task SendAsync(GrainId target, string route, TestMessage message); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); +} + +[GenerateSerializer] +public sealed record NamedFactoryMessagingSnapshot( + [property: Id(0)] Guid ActivationId, + [property: Id(1)] int OutboxCount, + [property: Id(2)] MessageEffect[] Effects); + +public sealed class NamedFactoryMessagingProbe +{ + private readonly ConcurrentDictionary> _completed = new(); + + public Task WaitAsync(GrainId grainId, CancellationToken cancellationToken) => + GetCompletion(grainId).Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + + public void Publish(GrainId grainId, NamedFactoryMessagingSnapshot snapshot) + { + if (snapshot.Effects.Length > 0 && snapshot.OutboxCount == 0) + { + GetCompletion(grainId).TrySetResult(snapshot); + } + } + + private TaskCompletionSource GetCompletion(GrainId grainId) => + _completed.GetOrAdd(grainId, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)); +} + +public sealed class NamedFactoryMessagingGrain : Grain, INamedFactoryMessagingGrain, IMessagingGrain, IInboxHandler +{ + private readonly IJournaledStateManager _owner; + private readonly IOutbox _outbox; + private readonly IDurableDictionary _effects; + private readonly SerializerSessionPool _sessions; + private readonly NamedFactoryMessagingProbe _probe; + private readonly Guid _activationId = Guid.NewGuid(); + + public NamedFactoryMessagingGrain( + IJournaledStateManager owner, + IInbox inbox, + IOutbox outbox, + [FromKeyedServices("cutover-effects")] IDurableDictionary effects, + SerializerSessionPool sessions, + NamedFactoryMessagingProbe probe) + { + _owner = owner; + _outbox = outbox; + _effects = effects; + _sessions = sessions; + _probe = probe; + inbox.Register(this); + } + + public async Task SendAsync(GrainId target, string route, TestMessage message) + { + var envelope = TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, route, message); + _outbox.Send(envelope); + await _owner.WriteStateAsync(); + } + + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); + + internal NamedFactoryMessagingSnapshot CaptureStorageWrite() => CreateSnapshot(); + internal void PublishStoredSnapshot(NamedFactoryMessagingSnapshot snapshot) => _probe.Publish(this.GetGrainId(), snapshot); + + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + + public bool CanHandle(IInboxHandlerContext context) => true; + + + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + var message = Assert.IsType(TestApplicationProtocol.Read(_sessions, context.Message).Body); + var outgoing = message.ForwardTo is { } target + ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, "messages/forwarded", + message with { ForwardTo = null }, context.Message.Envelope.MessageId.CreateChildKey("forwarded")) + : (OutboxMessage?)null; + + cancellationToken.ThrowIfCancellationRequested(); + _effects.TryGetValue(message.LogicalId, out var prior); + var effect = new MessageEffect(message.LogicalId, (prior?.Count ?? 0) + 1, message.Sequence, message.Value); + _effects[message.LogicalId] = effect; + if (outgoing is { } batch) + { + _outbox.Send(batch); + } + context.Complete(); + } + + private NamedFactoryMessagingSnapshot CreateSnapshot() => + new(_activationId, _outbox.Count(), _effects.Values.OrderBy(static effect => effect.Sequence).ToArray()); +} diff --git a/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs new file mode 100644 index 00000000000..083ce7a1fb2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +internal static class ReceiverTestServices +{ + private static readonly Func ReadCurrentContext = typeof(IGrainContext).Assembly + .GetType("Orleans.Runtime.RuntimeContext", throwOnError: true)! + .GetProperty("Current")!.GetMethod!.CreateDelegate>(); + + public static IGrainContext? CurrentGrainContext => ReadCurrentContext(); + + // Uses the production registration, replacing the application-facing outbox with a journaled capture. + public static void Add(IServiceCollection services, Action configure) + { + services.AddMessaging(configure); + services.RemoveAll(); + services.AddScoped(static sp => + new JournaledTestOutbox( + sp.GetRequiredKeyedService>("test-handler-output"))); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs new file mode 100644 index 00000000000..81be9cbe4f3 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs @@ -0,0 +1,98 @@ +using System.Collections.Concurrent; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class SnapshotProbe +{ + private readonly ConcurrentDictionary _latest = new(); + private readonly ConcurrentDictionary> _waiters = new(); + + internal int WaiterListCount => _waiters.Count; + + public async Task WaitAsync( + GrainId grainId, + Func predicate, + TimeSpan? timeout = null) + { + if (_latest.TryGetValue(grainId, out var current) && predicate(current)) + { + return current; + } + + var waiter = new Waiter(predicate); + List waiters; + while (true) + { + waiters = _waiters.GetOrAdd(grainId, static _ => []); + lock (waiters) + { + if (!_waiters.TryGetValue(grainId, out var currentWaiters) + || !ReferenceEquals(waiters, currentWaiters)) + { + continue; + } + + if (_latest.TryGetValue(grainId, out current) && predicate(current)) + { + RemoveWaiterListIfEmpty(grainId, waiters); + return current; + } + + waiters.Add(waiter); + break; + } + } + + try + { + return await waiter.Completion.Task.WaitAsync(timeout ?? TimeSpan.FromSeconds(30)); + } + finally + { + lock (waiters) + { + waiters.Remove(waiter); + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + } + + public void Publish(GrainId grainId, EndpointSnapshot snapshot) + { + _latest[grainId] = snapshot; + if (!_waiters.TryGetValue(grainId, out var waiters)) + { + return; + } + + lock (waiters) + { + foreach (var waiter in waiters.ToArray()) + { + if (waiter.Predicate(snapshot)) + { + waiters.Remove(waiter); + waiter.Completion.TrySetResult(snapshot); + } + } + + RemoveWaiterListIfEmpty(grainId, waiters); + } + } + + private void RemoveWaiterListIfEmpty(GrainId grainId, List waiters) + { + if (waiters.Count == 0) + { + _waiters.TryRemove(new KeyValuePair>(grainId, waiters)); + } + } + + private sealed class Waiter(Func predicate) + { + public Func Predicate { get; } = predicate; + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs new file mode 100644 index 00000000000..e796c52a077 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbeTests.cs @@ -0,0 +1,82 @@ +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class SnapshotProbeTests +{ + [Fact] + public async Task WaitAsync_TimeoutRemovesWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "timeout"); + var predicateCalls = 0; + + await Assert.ThrowsAsync( + () => probe.WaitAsync( + grainId, + _ => + { + predicateCalls++; + return false; + }, + TimeSpan.Zero)); + + Assert.Equal(0, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 0)); + + Assert.Equal(0, predicateCalls); + } + + [Fact] + public async Task WaitAsync_SuccessRemovesWaiterList() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "success"); + var wait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + var snapshot = CreateSnapshot(inboxCount: 1); + + probe.Publish(grainId, snapshot); + + Assert.Equal(0, probe.WaiterListCount); + Assert.Same(snapshot, await wait); + } + + [Fact] + public async Task WaitAsync_TimeoutRetainsListWithActiveWaiter() + { + var probe = new SnapshotProbe(); + var grainId = GrainId.Create("snapshot-probe", "shared-list"); + var activeWait = probe.WaitAsync(grainId, static snapshot => snapshot.InboxCount == 1); + + await Assert.ThrowsAsync( + () => probe.WaitAsync(grainId, static _ => false, TimeSpan.Zero)); + + Assert.Equal(1, probe.WaiterListCount); + + probe.Publish( + grainId, + CreateSnapshot(inboxCount: 1)); + + await activeWait; + Assert.Equal(0, probe.WaiterListCount); + } + + private static EndpointSnapshot CreateSnapshot(int inboxCount) => + new( + Guid.Empty, + string.Empty, + inboxCount, + 0, + 0, + [], + [], + [], + 0); +} diff --git a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs new file mode 100644 index 00000000000..c0f4cb4d52c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs @@ -0,0 +1,103 @@ +using System.Buffers; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Buffers; +using Orleans.Serialization.Codecs; +using Orleans.Serialization.Session; +using Orleans.Serialization.WireProtocol; + +namespace Orleans.Messaging.Tests.Support; + +// Routes and typed values belong to this test application, not to the messaging transport. +[GenerateSerializer] +public sealed record TestApplicationMessage( + [property: Id(0)] string Route, + [property: Id(1)] object? Body); + +internal static class TestApplicationProtocol +{ + public static HierarchicalKey NewMessageId() => HierarchicalKey.Create("tests", Guid.NewGuid().ToString("N")); + + public static OutboxMessage Create(SerializerSessionPool sessions, GrainId sender, GrainId receiver, + string route, object? body, HierarchicalKey messageId = default) => + new() + { + ReceiverId = receiver, + Envelope = CreateInbox(sessions, sender, route, body, messageId).Envelope + }; + + public static InboxMessage CreateInbox(SerializerSessionPool sessions, GrainId sender, + string route, object? body, HierarchicalKey messageId = default) + { + var key = messageId.IsDefault + ? body is TestMessage message ? message.LogicalId : NewMessageId() + : messageId; + return new() + { + Envelope = new() + { + MessageId = key, + Payload = Encode(sessions, new TestApplicationMessage(route, body)), + Subject = route, + Headers = new(StringComparer.Ordinal) + { + ["sender"] = new Serializer(sessions).SerializeToArray(sender) + } + } + }; + } + + public static byte[] Encode(SerializerSessionPool sessions, T value) + { + var output = new ArrayBufferWriter(); + using var session = sessions.GetSession(); + var writer = Writer.Create(output, session); + sessions.CodecProvider.GetCodec().WriteField(ref writer, 0, typeof(T), value); + writer.Commit(); + return output.WrittenSpan.ToArray(); + } + + public static T Decode(SerializerSessionPool sessions, byte[] payload) + { + using var session = sessions.GetSession(); + var reader = Reader.Create(payload, session); + var field = reader.ReadFieldHeader(); + return sessions.CodecProvider.GetCodec().ReadValue(ref reader, field)!; + } + + public static TestApplicationMessage Read(SerializerSessionPool sessions, InboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static TestApplicationMessage Read(SerializerSessionPool sessions, OutboxMessage message) => + Decode(sessions, message.Envelope.Payload); + + public static GrainId Sender(SerializerSessionPool sessions, Envelope envelope) => + envelope.Headers is { } headers && headers.TryGetValue("sender", out var sender) + ? new Serializer(sessions).Deserialize(sender) : default; + + public static string? Subject(Envelope envelope) => + envelope.Subject; + + public static InboxMessage With(InboxMessage message, SerializerSessionPool sessions, + HierarchicalKey? messageId = null, GrainId? sender = null, string? subject = null, + byte[]? payload = null, bool removeSender = false, bool removeSubject = false) + { + var headers = message.Envelope.Headers is { } existing + ? new Dictionary(existing, StringComparer.Ordinal) : null; + if (removeSender) headers?.Remove("sender"); + if (sender is { } value) + { + (headers ??= new(StringComparer.Ordinal))["sender"] = new Serializer(sessions).SerializeToArray(value); + } + return new() + { + Envelope = new() + { + MessageId = messageId ?? message.Envelope.MessageId, + Payload = payload ?? message.Envelope.Payload, + Subject = removeSubject ? null : subject ?? message.Envelope.Subject, + Headers = headers + } + }; + } +} From 732fab571deea0587284eed648cde2b5b3f1b854 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 15:18:42 -0700 Subject: [PATCH 29/95] refactor(journaling): remove operation hooks Messaging relies only on IStateMachine.PrepareWriteAsync and JournaledStatePreCommitException. Removes IJournaledStateHook, JournaledStateOperation, AddHookAsync/RemoveHookAsync, and JournaledStatePostCommitException, restoring their consumers to main. --- .../CompatibilitySuppressions.xml | 28 - src/Orleans.Journaling/IJournaledStateHook.cs | 63 - .../IJournaledStateManager.cs | 51 +- src/Orleans.Journaling/IStateMachine.cs | 2 +- .../JournaledStateManager.Hooks.cs | 105 -- .../JournaledStateManager.cs | 380 ++-- .../JournaledStatePostCommitException.cs | 22 - .../JournaledStatePreCommitException.cs | 11 +- src/Orleans.Journaling/README.md | 52 +- src/Orleans.Messaging/Inbox.cs | 4 - .../Orleans.Journaling/Orleans.Journaling.cs | 49 +- .../DurableListJournalBenchmarks.cs | 17 - .../JournaledJobShardDiscoveryTests.cs | 4 - .../CodecRecoveryTests.cs | 24 +- .../DurableCollectionDirectWriteTests.cs | 17 - .../DurableListDirectWriteTests.cs | 17 - .../DurableStateManagerTests.cs | 2 +- .../KeyedJournalingRegistrationTests.cs | 4 - .../StateManagerHookTests.cs | 1566 ----------------- .../StateManagerPreparationTests.cs | 61 +- .../Functional/InboxHandlerCompletionTests.cs | 29 +- .../PublicMessagingRegistrationTests.cs | 2 - .../Support/JournalProbeState.cs | 24 + .../Support/MessagingTestGrains.cs | 25 +- 24 files changed, 237 insertions(+), 2322 deletions(-) delete mode 100644 src/Orleans.Journaling/IJournaledStateHook.cs delete mode 100644 src/Orleans.Journaling/JournaledStateManager.Hooks.cs delete mode 100644 src/Orleans.Journaling/JournaledStatePostCommitException.cs delete mode 100644 test/Orleans.Journaling.Tests/StateManagerHookTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/JournalProbeState.cs diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index 5e86005fc7b..eb687e7cf91 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -239,13 +239,6 @@ lib/net8.0/Orleans.Journaling.dll true - - CP0006 - M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) - lib/net10.0/Orleans.Journaling.dll - lib/net10.0/Orleans.Journaling.dll - true - CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -253,13 +246,6 @@ lib/net10.0/Orleans.Journaling.dll true - - CP0006 - M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) - lib/net10.0/Orleans.Journaling.dll - lib/net10.0/Orleans.Journaling.dll - true - CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) @@ -288,13 +274,6 @@ lib/net10.0/Orleans.Journaling.dll true - - CP0006 - M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) - lib/net8.0/Orleans.Journaling.dll - lib/net8.0/Orleans.Journaling.dll - true - CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -302,13 +281,6 @@ lib/net8.0/Orleans.Journaling.dll true - - CP0006 - M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) - lib/net8.0/Orleans.Journaling.dll - lib/net8.0/Orleans.Journaling.dll - true - CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs deleted file mode 100644 index d8395a31615..00000000000 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ /dev/null @@ -1,63 +0,0 @@ -namespace Orleans.Journaling; - -/// -/// Identifies a journal persistence operation. -/// -public enum JournaledStateOperation -{ - /// Appends pending journal entries. - Write, - - /// Replaces storage with a snapshot. - Snapshot, - - /// Deletes storage and resets registered states. - Delete -} - -/// -/// Participates in the prerequisites and completion of actual journal operations. -/// -/// -/// Before hooks run in reverse list order on the owner's logical execution context, outside its lock. -/// The hook at index zero runs last among before hooks. Write operations then prepare the registered -/// states before capture; deletion proceeds directly from the before hooks. -/// Prerequisites remain valid until capture or deletion, covering changes staged during asynchronous -/// preparation and while the journal work loop resumes. After hooks run in forward list order after -/// storage acknowledgement and state acknowledgement or reset, including successful writes which -/// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later -/// pending changes separate. Hook implementations must complete without calling initialization, -/// persistence, or disposal on their own journal owner. Those operations are serialized behind the -/// current operation, so awaiting them from a hook would create a circular dependency. -/// Dependencies between journal owners must also remain acyclic. -/// Callbacks can request a hook registration change, observing its queued completion outside the -/// callback rather than awaiting work which follows the current operation. -/// -public interface IJournaledStateHook -{ - /// - /// Establishes prerequisites before capture or storage deletion. - /// - /// The operation about to execute. - /// The token for the owned operation's lifetime. - /// A completion representing the prerequisite work. - /// - /// Failure reports and retains pending changes - /// for an explicit retry. All staged changes remain safe to commit. Full deletion requires the - /// owner to stop admission and drain feature operations before queuing deletion. - /// - ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; - - /// - /// Completes post-persistence work after the operation succeeds. - /// - /// The completed operation. - /// The token for the owned operation's lifetime. - /// A completion representing post-persistence work. - /// - /// Every after hook is invoked even when an earlier after hook fails. Failures are surfaced as - /// and leave the successfully persisted owner usable. - /// Durable feature state supplies recovery for interrupted post-persistence work. - /// - ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; -} diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 9bce3b22a4b..d46d1bce118 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -8,44 +8,10 @@ namespace Orleans.Journaling; /// /// The owner registers state machines and initializes the journal before using recovered state. /// State machine instances and their dependencies retain the lifetime assigned by their caller. -/// Disposal drains owned storage and hook operations before releasing resources. +/// Disposing this manager stops journal processing and releases its journal resources. /// public interface IJournaledStateManager : IAsyncDisposable { - /// - /// Adds a hook before the next persistence operation which follows this registration. - /// - /// The hook to register. - /// The insertion index, or to append. - /// The token used to cancel the caller's wait. - /// A completion representing application of the registration. - /// - /// Registration completes synchronously before initialization begins, supporting constructor registration. - /// After initialization begins, idle registration applies synchronously; otherwise it is queued - /// between owned operations, after the current operation's after hooks finish. - /// Before hooks execute in reverse registration order; after hooks execute in forward order. - /// Register the final before hook at index zero. Writes then prepare the registered states before capture. - /// Registration is independent of state-machine registration and persists through recovery and deletion. - /// Every implementation provides registration and invokes hooks at the documented operation boundaries. - /// Queued registration is a write-coalescing barrier. - /// Cancellation ends the caller's wait while admitted registration remains owned by the manager. - /// Synchronous state callbacks can request registration and observe its completion outside the callback. - /// Await registration before staging changes whose safety depends on the added hook. - /// - ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default); - - /// - /// Removes a hook between persistence operations. - /// - /// The registered hook to remove. - /// The token used to cancel the caller's wait. - /// A completion representing application of the removal. - /// The hook is not registered when the removal is applied. - /// - /// Removal has the same ownership and ordering guarantees as . - /// - ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default); - /// ValueTask IAsyncDisposable.DisposeAsync() => default; @@ -85,13 +51,11 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Stage mutations only after establishing that they are safe to commit. Pending changes are shared /// by all callers using this manager. Storage acknowledgement establishes durability. - /// The manager invokes each state's after before hooks, - /// completing a synchronous ready pass before capture. - /// A storage or state-processing failure fences the manager; recovery requires a new manager and state machine instances. + /// A failed journal operation fences the manager; recovery requires a new manager and state machine instances. + /// Before capture, the manager prepares each registered state through . + /// A preparation failure reports and retains pending changes + /// for an explicit retry. /// Cancellation stops the caller's wait; an already queued write continues to its storage outcome. - /// State preparation or before-hook failure reports and retains pending - /// changes for an explicit retry. After-hook failure reports - /// after successful persistence. /// /// The token used to cancel the caller's wait. /// A task representing the write acknowledgement. @@ -103,10 +67,7 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. - /// A storage or state-reset failure permanently fences the manager and requests deactivation of its owning grain. - /// Before-hook failure reports with state retained. - /// After-hook failure reports after storage deletion - /// and state reset succeed. + /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. /// /// The cancellation token. /// A which represents the operation. diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index f0c07c397ff..70a24630e38 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -75,7 +75,7 @@ void OnRecoveryCompleted() { } /// The token which cancels preparation when the journal owner stops. /// A completion representing readiness to capture this state. /// - /// The manager invokes preparation after before hooks and before capturing an append or snapshot. + /// The manager invokes preparation before capturing an append or snapshot. /// Preparation is idempotent and prepares this state's own prerequisites. After asynchronous /// preparation completes, the manager repeats preparation for all states to include changes /// staged on interleaved turns. Prepared prerequisites complete synchronously on subsequent calls diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs deleted file mode 100644 index 5b5c68fd943..00000000000 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ /dev/null @@ -1,105 +0,0 @@ -using System.Runtime.ExceptionServices; - -namespace Orleans.Journaling; - -internal partial class JournaledStateManager -{ - private List? _hooks; - - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => - MutateHookAsync(hook, index, remove: false, cancellationToken); - - public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => - MutateHookAsync(hook, index: null, remove: true, cancellationToken); - - private ValueTask MutateHookAsync(IJournaledStateHook hook, int? index, bool remove, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(hook); - cancellationToken.ThrowIfCancellationRequested(); - Task task; - lock (_lock) - { - ThrowIfFenced(); - ObjectDisposedException.ThrowIf(_disposed != 0, this); - _shutdownCancellation.Token.ThrowIfCancellationRequested(); - if (_workLoop is null || (!_workItemRunning && _workQueue.Count == 0)) - { - ApplyHookMutation(hook, index, remove); - return default; - } - - var workItem = new HookMutationWorkItem(hook, index, remove); - _workQueue.Enqueue(workItem); - task = workItem.Task; - } - - _workSignal.Signal(); - return new(task.WaitAsync(cancellationToken)); - } - - private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove) - { - var hooks = _hooks ??= new(); - if (remove) - { - if (!hooks.Remove(hook)) - { - throw new InvalidOperationException("The journal operation hook is not registered."); - } - } - else - { - hooks.Insert(index ?? hooks.Count, hook); - } - } - - private ValueTask InvokeBeforeHooksAsync(ref int index, JournaledStateOperation operation, CancellationToken cancellationToken) - { - while (index >= 0) - { - cancellationToken.ThrowIfCancellationRequested(); - var pending = _hooks![index--].BeforeOperationAsync(operation, cancellationToken); - if (!pending.IsCompleted) - { - return pending; - } - - pending.GetAwaiter().GetResult(); - } - - return default; - } - - private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - List? failures = null; - for (var i = 0; i < _hooks!.Count; i++) - { - try - { - await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - catch (Exception exception) - { - (failures ??= []).Add(exception); - } - } - - if (failures is { Count: 1 }) - { - ExceptionDispatchInfo.Capture(failures[0]).Throw(); - } - - if (failures is not null) - { - throw new AggregateException(failures); - } - } - - private sealed class HookMutationWorkItem(IJournaledStateHook hook, int? index, bool remove) : WorkItem - { - public IJournaledStateHook Hook { get; } = hook; - public int? Index { get; } = index; - public bool Remove { get; } = remove; - } -} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 931f9bc6d21..d3dbe0a7c97 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1,7 +1,6 @@ using System.Buffers; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using Orleans.Diagnostics; using Orleans.Serialization.Buffers; @@ -29,15 +28,11 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private readonly StateDirectory _journalStreamDirectory; private readonly RetiredStateTracker _retirementTracker; private Task? _workLoop; - // Under _lock, covers recovery and dequeued work through the last after callback. - private bool _workItemRunning; private ManagerState _state; private long _deletionGeneration; private Exception? _failure; - // A snapshot requirement not yet claimed by a write. Consume under the lock before preparation awaits. - private bool _snapshotRequired; + private bool _migrationSnapshotRequired; private int _disposed; - private Task? _disposeTask; public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) : this(shared, CreateStorage(storageProvider, CreateJournalId(grainContext)), grainContext) @@ -103,8 +98,6 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal string WriteJournalFormatKey => _shared.JournalFormatKey; - private bool IsSnapshotRequired => _snapshotRequired || _storage.IsCompactionRequested; - internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine) @@ -134,7 +127,7 @@ protected void EnsureRegistrationAllowed() _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (_workLoop is not null) { - throw new InvalidOperationException("Journal registrations cannot be changed after journaled state manager initialization has begun."); + throw new InvalidOperationException("New states cannot be registered after journaled state manager initialization has begun."); } } } @@ -171,11 +164,10 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def ThrowIfFenced(); if (_workLoop is null) { - _workItemRunning = true; _workLoop = Start(); } - task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; + task = EnqueueOrGetPendingWorkItem(out didEnqueue); } if (didEnqueue) @@ -197,12 +189,6 @@ private async Task WorkLoop() await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.ForceYielding); while (!_shutdownCancellation.IsCancellationRequested) { - lock (_lock) - { - _workItemRunning = true; - } - - Exception? recoveryFailure = null; try { await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -215,17 +201,15 @@ private async Task WorkLoop() } catch (Exception exception) { - recoveryFailure = exception; - LogErrorProcessingWorkItems(_shared.Logger, exception); - } - finally - { - lock (_lock) + try { - _workItemRunning = false; - if (recoveryFailure is not null) + LogErrorProcessingWorkItems(_shared.Logger, exception); + } + finally + { + lock (_lock) { - FaultQueuedWorkItemsUnderLock(recoveryFailure); + FaultQueuedWorkItemsUnderLock(exception); } } } @@ -261,11 +245,6 @@ private async Task WorkLoop() WorkItem workItem; lock (_lock) { - if (_shutdownCancellation.IsCancellationRequested) - { - return; - } - if (!_workQueue.TryDequeue(out var dequeuedWorkItem)) { // Wait for the queue to be signaled again. @@ -273,88 +252,84 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; - _workItemRunning = true; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); - string? queueOperation = null; - TimeSpan queueDuration = default; + var queueOperation = GetStorageQueueOperation(workItem); + var queueDuration = queueOperation is null + ? default + : _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); var recordQueueDuration = workItem is DeleteStateWorkItem; - Activity? storageActivity = null; + Activity? storageActivity = queueOperation is null + ? null + : ActivitySources.StorageGrainSource.StartActivity( + $"journal {queueOperation}", + ActivityKind.Internal, + workItem.TraceParent, + tags: null, + links: workItem.TraceLinks); + if (storageActivity is not null) + { + storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); + } - JournaledStateOperation? operationType = null; - string? compactionReason = null; - var beforeCommitRunning = false; - var afterHookRunning = false; try { - lock (_lock) + if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem + && workItem.DeletionGeneration != Volatile.Read(ref _deletionGeneration)) { - if (workItem is WriteStateWorkItem - && workItem.DeletionGeneration != _deletionGeneration) - { - throw new InvalidOperationException( - "The journaled state operation was queued before deletion reset its state."); - } - - operationType = GetOperationType(workItem); - if (operationType == JournaledStateOperation.Snapshot) - { - compactionReason = _snapshotRequired - ? JournalingInstruments.CompactionReasonUserSnapshot - : JournalingInstruments.CompactionReasonStorageRequested; - _snapshotRequired = false; - } + throw new InvalidOperationException( + "The journaled state operation was queued before deletion reset its state."); } - (queueOperation, queueDuration, storageActivity) = GetStorageQueueOperation(workItem, operationType, processingTimestamp); - - if (operationType is { } operation && _hooks is { Count: > 0 }) + if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem) { - beforeCommitRunning = true; - // Keep synchronous final completion and capture in the same work-loop continuation. - for (var i = _hooks.Count - 1; i >= 0;) + try { - await InvokeBeforeHooksAsync(ref i, operation, _shutdownCancellation.Token).ConfigureAwait(true); - } - - _shutdownCancellation.Token.ThrowIfCancellationRequested(); - beforeCommitRunning = false; - } + for (var prepared = false; !prepared;) + { + await PrepareStates(ref prepared, _shutdownCancellation.Token).ConfigureAwait(true); + } - if (workItem is WriteStateWorkItem) - { - beforeCommitRunning = true; - for (var prepared = false; !prepared;) + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) + { + workItem.TrySetCanceled(_shutdownCancellation.Token); + return; + } + catch (Exception exception) { - await PrepareStates(ref prepared, _shutdownCancellation.Token).ConfigureAwait(true); + LogStatePreparationFailed(_shared.Logger, exception); + storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal state preparation failed."); + workItem.SetException(new JournaledStatePreCommitException(exception)); + continue; } - _shutdownCancellation.Token.ThrowIfCancellationRequested(); - beforeCommitRunning = false; } // Note that the implementation of each command is inlined to avoid allocating unnecessary async states. // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { - case HookMutationWorkItem mutation: - { - // The work loop owns the hook set while _workItemRunning is true. - ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); - break; - } - - case WriteStateWorkItem: + case AppendJournalWorkItem: + case WriteSnapshotWorkItem: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. - var isSnapshot = operationType == JournaledStateOperation.Snapshot; + var isSnapshot = workItem is WriteSnapshotWorkItem + || _migrationSnapshotRequired + || _storage.IsCompactionRequested; var operationLabel = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; if (isSnapshot) { - _shared.Instruments.OnCompactionTriggered(compactionReason!); + var compactionReason = workItem is WriteSnapshotWorkItem + ? JournalingInstruments.CompactionReasonUserSnapshot + : _migrationSnapshotRequired + ? JournalingInstruments.CompactionReasonMigration + : JournalingInstruments.CompactionReasonStorageRequested; + _shared.Instruments.OnCompactionTriggered(compactionReason); } _shared.Instruments.OnWriteCoalesced(operationLabel, workItem.CallerCount); var gatherStartTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -375,6 +350,11 @@ private async Task WorkLoop() RetireOrResurectStates(); } + if (_migrationSnapshotRequired) + { + ThrowIfMigrationBlockedByRetiredStates(); + } + // The map of state ids is itself stored as a durable state with the id 0. // This must be stored first, since it includes the identities of all other states, which are needed when replaying the journal. // If we removed retired states, this snapshot will persist that change. @@ -515,6 +495,11 @@ private async Task WorkLoop() { state.OnWriteCompleted(); } + + if (isSnapshot) + { + _migrationSnapshotRequired = false; + } } } @@ -580,64 +565,13 @@ private async Task WorkLoop() } } - if (operationType is { } completedOperation && _hooks is { Count: > 0 }) - { - afterHookRunning = true; - await InvokeAfterHooksAsync(completedOperation, _shutdownCancellation.Token).ConfigureAwait(true); - afterHookRunning = false; - } - if (recordQueueDuration && queueOperation is not null) { _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: true); } storageActivity?.SetStatus(ActivityStatusCode.Ok); - lock (_lock) - { - _workItemRunning = false; - workItem.SetResult(); - } - } - catch (Exception exception) when (beforeCommitRunning || afterHookRunning) - { - if (beforeCommitRunning && operationType == JournaledStateOperation.Snapshot) - { - lock (_lock) - { - _snapshotRequired = true; - } - } - - if (beforeCommitRunning && exception is OperationCanceledException && _shutdownCancellation.IsCancellationRequested) - { - workItem.TrySetCanceled(_shutdownCancellation.Token); - continue; - } - - LogOperationHookFailed(_shared.Logger, exception, operationType!.Value, afterHookRunning); - storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal operation callback failed."); - if (recordQueueDuration && queueOperation is not null) - { - _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); - } - - lock (_lock) - { - _workItemRunning = false; - workItem.SetException(afterHookRunning - ? new JournaledStatePostCommitException(operationType.Value, exception) - : new JournaledStatePreCommitException(operationType.Value, exception)); - } - } - catch (Exception exception) when (workItem is HookMutationWorkItem) - { - LogHookMutationFailed(_shared.Logger, exception); - lock (_lock) - { - _workItemRunning = false; - workItem.SetException(exception); - } + workItem.SetResult(); } catch (Exception exception) { @@ -671,11 +605,6 @@ private async Task WorkLoop() } finally { - lock (_lock) - { - _workItemRunning = false; - } - storageActivity?.Dispose(); } } @@ -692,14 +621,7 @@ private async Task WorkLoop() } } - private JournaledStateOperation? GetOperationType(WorkItem workItem) => workItem switch - { - WriteStateWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, - WriteStateWorkItem => JournaledStateOperation.Write, - DeleteStateWorkItem => JournaledStateOperation.Delete, - _ => null - }; - + // Completes when every state is synchronously ready; otherwise returns the first incomplete preparation. private ValueTask PrepareStates(ref bool prepared, CancellationToken cancellationToken) { prepared = true; @@ -713,9 +635,11 @@ private ValueTask PrepareStates(ref bool prepared, CancellationToken cancellatio prepared = false; return preparation; } + preparation.GetAwaiter().GetResult(); } } + return ValueTask.CompletedTask; } @@ -791,6 +715,20 @@ private void RetireOrResurectStates() } } + private void ThrowIfMigrationBlockedByRetiredStates() + { + foreach (var state in _statesMap.Values) + { + if (state is RetiredStateMachine { PreservedEntries.Count: > 0 } retiredState) + { + throw new InvalidOperationException( + $"Cannot migrate journal to format key '{_shared.JournalFormatKey}' because stream " + + $"{retiredState.StreamId.Value} belongs to a state which is not currently registered. Register the state so it can be decoded " + + "and snapshotted in the configured format, or keep the previous journal format configured until the state can be retired."); + } + } + } + private JournalStreamWriter CreateJournalStreamWriter(JournalStreamId streamId) => _journalWriter.CreateJournalStreamWriter(streamId); private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWriter, bool isSnapshot, uint id, IStateMachine state) @@ -814,7 +752,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = de lock (_lock) { ThrowIfStateOperationsUnavailable(); - task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; + task = EnqueueOrGetPendingWorkItem(out didEnqueue); } if (didEnqueue) @@ -889,7 +827,7 @@ private async Task RecoverAsync(CancellationToken cancellationToken) private void ResetForRecovery() { _journalWriter.Reset(); - _snapshotRequired = false; + _migrationSnapshotRequired = false; _statesMap.Clear(); _statesMap[StateDirectory.Id] = _journalStreamDirectory; _statesMap[RetiredStateTracker.Id] = _retirementTracker; @@ -960,7 +898,7 @@ private void ProcessRecoveryBuffer(JournalBufferReader buffer, IJournalMetadata? { if (!string.Equals(journalFormatKey, _shared.JournalFormatKey, StringComparison.Ordinal)) { - _snapshotRequired = true; + _migrationSnapshotRequired = true; } var journalFormat = GetJournalFormat(journalFormatKey); @@ -1003,9 +941,11 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); - _snapshotRequired = IsSnapshotRequired; - operation = _snapshotRequired ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; - pendingWrite = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; + var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; + operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; + pendingWrite = isSnapshot + ? EnqueueOrGetPendingWorkItem(out didEnqueue) + : EnqueueOrGetPendingWorkItem(out didEnqueue); } if (didEnqueue) @@ -1106,39 +1046,26 @@ private async ValueTask DeleteStorageAsync(CancellationToken cancellationToken) } } - private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) + private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) where TWorkItem : WorkItem, new() { - TWorkItem? pending = null; foreach (var workItem in _workQueue) { - if (workItem is HookMutationWorkItem) - { - // Later callers must not coalesce into work which precedes a registration change. - pending = null; - continue; - } - - if (pending is not null || workItem is not TWorkItem candidate) + if (workItem.GetType() != typeof(TWorkItem)) { continue; } - if (workItem is WriteStateWorkItem + if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem && workItem.DeletionGeneration != _deletionGeneration) { continue; } - pending = candidate; - } - - if (pending is not null) - { - pending.RecordTraceContext(); - pending.AddCaller(); + workItem.RecordTraceContext(); + workItem.AddCaller(); didEnqueue = false; - return pending; + return workItem.Task; } var newWorkItem = new TWorkItem(); @@ -1147,7 +1074,7 @@ private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) newWorkItem.RecordTraceContext(); _workQueue.Enqueue(newWorkItem); didEnqueue = true; - return newWorkItem; + return newWorkItem.Task; } private void BindState(string name, uint id) @@ -1182,22 +1109,9 @@ void ILifecycleParticipant.Participate(IGrainLifecycle observer private async Task StopAsync(CancellationToken cancellationToken) { - AggregateException? cancellationFailure = null; lock (_lock) { - try - { - _shutdownCancellation.Cancel(); - } - catch (AggregateException exception) - { - cancellationFailure = exception; - } - } - - if (cancellationFailure is not null) - { - LogShutdownCancellationFailed(_shared.Logger, cancellationFailure); + _shutdownCancellation.Cancel(); } _workSignal.Signal(); @@ -1212,11 +1126,6 @@ private async Task StopAsync(CancellationToken cancellationToken) { CancelQueuedWorkItems(_shutdownCancellation.Token); } - - if (cancellationFailure is not null) - { - ExceptionDispatchInfo.Capture(cancellationFailure).Throw(); - } } private void CancelQueuedWorkItems(CancellationToken cancellationToken) @@ -1235,41 +1144,21 @@ void IDisposable.Dispose() DisposeAsync().AsTask().GetAwaiter().GetResult(); } - public ValueTask DisposeAsync() + public async ValueTask DisposeAsync() { - lock (_lock) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { - if (_disposeTask is null) - { - var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - _disposeTask = completion.Task; - _ = DisposeCoreAsync(completion); - } - - return new(_disposeTask); + return; } - } - private async Task DisposeCoreAsync(TaskCompletionSource completion) - { - _disposed = 1; try { - try - { - await StopAsync(CancellationToken.None).ConfigureAwait(false); - } - finally - { - _shutdownCancellation.Dispose(); - _journalWriter.Dispose(); - } - - completion.SetResult(); + await StopAsync(CancellationToken.None).ConfigureAwait(false); } - catch (Exception exception) + finally { - completion.SetException(exception); + _shutdownCancellation.Dispose(); + _journalWriter.Dispose(); } } @@ -1351,37 +1240,20 @@ public void RecordTraceContext() } } - private (string? Operation, TimeSpan QueueDuration, Activity? Activity) GetStorageQueueOperation( - WorkItem workItem, - JournaledStateOperation? operationType, - long processingTimestamp) - { - var operation = operationType switch + private static string? GetStorageQueueOperation(WorkItem workItem) => + workItem switch { - JournaledStateOperation.Snapshot => JournalingInstruments.OperationSnapshot, - JournaledStateOperation.Write => JournalingInstruments.OperationAppend, - JournaledStateOperation.Delete => JournalingInstruments.OperationDelete, + AppendJournalWorkItem => JournalingInstruments.OperationAppend, + WriteSnapshotWorkItem => JournalingInstruments.OperationSnapshot, + DeleteStateWorkItem => JournalingInstruments.OperationDelete, _ => null }; - if (operation is null) - { - return default; - } - - var queueDuration = _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); - var activity = ActivitySources.StorageGrainSource.StartActivity( - $"journal {operation}", - ActivityKind.Internal, - workItem.TraceParent, - tags: null, - links: workItem.TraceLinks); - activity?.SetTag(ActivityTagKeys.JournalStorageOperation, operation); - return (operation, queueDuration, activity); - } private sealed class InitializeWorkItem : WorkItem; - private sealed class WriteStateWorkItem : WorkItem; + private sealed class AppendJournalWorkItem : WorkItem; + + private sealed class WriteSnapshotWorkItem : WorkItem; private sealed class DeleteStateWorkItem : WorkItem; @@ -1505,6 +1377,8 @@ private sealed class RetiredStateMachine(JournalStreamId streamId) : IStateMachi public JournalStreamId StreamId { get; } = streamId; + public IReadOnlyList PreservedEntries => _preservedEntries; + void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => _preservedEntries.Add(new PreservedJournalEntry(entry.FormatKey, entry.Reader)); @@ -1527,18 +1401,8 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } [LoggerMessage( Level = LogLevel.Error, - Message = "Journal {Operation} callback failed. Persistence completed: {Committed}.")] - private static partial void LogOperationHookFailed(ILogger logger, Exception exception, JournaledStateOperation operation, bool committed); - - [LoggerMessage( - Level = LogLevel.Error, - Message = "Journal operation hook registration change failed.")] - private static partial void LogHookMutationFailed(ILogger logger, Exception exception); - - [LoggerMessage( - Level = LogLevel.Error, - Message = "Journal shutdown cancellation callback failed; owned operations are drained before resources are released.")] - private static partial void LogShutdownCancellationFailed(ILogger logger, Exception exception); + Message = "Journal state preparation failed; pending changes remain staged.")] + private static partial void LogStatePreparationFailed(ILogger logger, Exception exception); [LoggerMessage( Level = LogLevel.Information, diff --git a/src/Orleans.Journaling/JournaledStatePostCommitException.cs b/src/Orleans.Journaling/JournaledStatePostCommitException.cs deleted file mode 100644 index 8274c8726d6..00000000000 --- a/src/Orleans.Journaling/JournaledStatePostCommitException.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Orleans.Journaling; - -/// -/// Reports a hook failure after the journal operation and its state acknowledgement or reset succeeded. -/// -/// -/// The journal owner remains usable. The caller handles the failed post-persistence work using -/// the feature's durable recovery protocol, preserving the completed business operation. -/// -[GenerateSerializer] -public sealed class JournaledStatePostCommitException : Exception -{ - /// - /// Initializes a post-persistence hook failure. - /// - /// The successfully completed journal operation. - /// The failure or aggregate of failures from after hooks. - public JournaledStatePostCommitException(JournaledStateOperation operation, Exception innerException) - : base($"Journal operation '{operation}' completed, but a post-persistence hook failed.", innerException) - { - } -} diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs index e4ddb71939e..55a0cbbbae5 100644 --- a/src/Orleans.Journaling/JournaledStatePreCommitException.cs +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -1,7 +1,7 @@ namespace Orleans.Journaling; /// -/// Reports a persistence prerequisite failure which prevented the journal storage operation. +/// Reports a state preparation failure which prevented a journal write. /// /// /// Pending changes remain staged and safe to commit. The caller can restore the prerequisite @@ -11,12 +11,11 @@ namespace Orleans.Journaling; public sealed class JournaledStatePreCommitException : Exception { /// - /// Initializes a persistence prerequisite failure. + /// Initializes a state preparation failure. /// - /// The journal operation prevented by the failed prerequisite. - /// The original prerequisite failure. - public JournaledStatePreCommitException(JournaledStateOperation operation, Exception innerException) - : base($"Journal operation '{operation}' was prevented by a persistence prerequisite failure.", innerException) + /// The original preparation failure. + public JournaledStatePreCommitException(Exception innerException) + : base("The journal write was prevented by a state preparation failure.", innerException) { } } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index d711ebee8bf..d829288dcb2 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -222,7 +222,7 @@ Recovery resets state machines and replays durable entries. A failed initializat same manager and registered states. `JournalReplayContext.ResolveStateMachine` routes entries to the state machine for their stream. -Write preparation runs after before hooks and before append or snapshot capture. +Write preparation runs before append or snapshot capture. Each state prepares its own prerequisites idempotently. An asynchronous completion leaves prepared prerequisites synchronously ready; the manager then rechecks all states to include mutations staged on interleaved turns. A complete synchronous pass flows @@ -278,56 +278,6 @@ the operation's token before use, or deliberately supply them through a registra lifecycle ownership. Creation through the explicit-journal factory keeps failure handling independent of the ambient grain context, including when the caller subsequently enrolls the manager in a lifecycle. -## Journal operation hooks - -Use `IJournaledStateManager.AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to register -`IJournaledStateHook` callbacks. Registration completes synchronously before initialization, -supporting constructors. After initialization begins, changes apply synchronously when idle, -or queue between complete journal operations while busy. -Each operation keeps its hook set through preparation, capture, storage ACK and after callbacks. -Queued changes are write-coalescing barriers: writes admitted afterward observe the changed set. -Caller cancellation ends only its wait for an admitted change; the manager retains ownership. -Await registration before staging changes whose safety depends on the new hook. -Every journal owner implementation provides registration and invokes callbacks at the operation -boundaries described below. Delegating owners forward registration methods to their inner owner. -Synchronous state callbacks can request a queued registration and observe completion outside the callback. - -For each actual write, snapshot, or deletion, before hooks run in reverse list order outside -the manager lock. Add the final before hook with `index: 0`. Writes then prepare the -registered states before capture; deletion follows the before hooks. Prerequisites remain valid through capture or -deletion, covering changes staged during asynchronous preparation and while the work loop resumes. -When state preparation is synchronously ready, capture follows in the same work-loop -continuation. Other owner turns can stage changes while an incomplete callback is awaited, -so its prerequisite covers those changes through capture or deletion. -Keep operation-local bookkeeping for the captured batch separate from later pending changes. - -After hooks run in list order after storage acknowledgement and state acknowledgement or reset. -They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual -operation. Features implement the before and after callbacks on their own hook, retaining their -identity and operation-local bookkeeping there. -Queued writes coalesce into one operation. A snapshot requirement from any caller upgrades that -pending work to a snapshot, even if the storage compaction signal clears before it runs. -The manager consumes the pending requirement when selecting a write and restores it when a before -hook or state preparation fails. Storage acknowledgement leaves newer snapshot requests untouched. - -| Outcome | Owner and caller behavior | -| --- | --- | -| Before hook or state preparation fails | `JournaledStatePreCommitException` retains pending state for an explicit retry after restoring the prerequisite. | -| Storage or state processing fails | The original failure fences the manager; create a fresh owner and recover the durable outcome. | -| After hook fails | `JournaledStatePostCommitException` reports successful persistence. Every remaining after hook runs, failures are aggregated, and the manager stays usable. The feature's durable recovery protocol resumes interrupted post-persistence work. | - -Hooks receive the owner's shutdown token. Caller cancellation ends the caller's wait while -the owned prerequisite, capture, storage, and completion phases continue. Disposal cancels the -owner token and drains owned work before releasing journal resources, including when cancellation -callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. -Shutdown closes work admission and cancels queued operations while an already running operation -drains to its actual storage and hook outcome. -Hook implementations complete without calling initialization, persistence, or disposal on their -own owner: awaiting an operation serialized behind the current callback creates a circular dependency. -Dependencies between journal owners also remain acyclic. -Before whole-journal deletion, the feature owner stops admission and drains its own operations; -deletion completion follows storage deletion and registered-state reset. - ## State identity and retirement Preserve state names across activations and deployments. A stream absent from the setup declarations diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 96eaaa322b7..f0eaef7c39a 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -324,10 +324,6 @@ private async ValueTask PersistAsync() { await _pump.StateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); } - catch (JournaledStatePostCommitException) - { - throw; - } catch (Exception exception) { _pump.Fail(exception, "Inbox persistence failed."); diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 516ebaf4d04..ae3d5f361d8 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,21 +219,13 @@ public partial interface IDurableValue T? Value { get; set; } } - public partial interface IJournaledStateHook - { - System.Threading.Tasks.ValueTask AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); - System.Threading.Tasks.ValueTask BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); - } - public partial interface IJournaledStateManager : System.IAsyncDisposable { long PendingWriteByteCount { get; } - System.Threading.Tasks.ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken = default); void RegisterStateMachine(string name, IStateMachine stateMachine); - System.Threading.Tasks.ValueTask RemoveHookAsync(IJournaledStateHook hook, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); bool TryGetStateMachine(string name, out IStateMachine? stateMachine); System.Threading.Tasks.ValueTask WriteStateAsync(System.Threading.CancellationToken cancellationToken = default); @@ -433,23 +425,10 @@ public sealed partial class JournaledStateManagerOptions public System.TimeSpan RetirementGracePeriod { get { throw null; } set { } } } - public enum JournaledStateOperation - { - Write = 0, - Snapshot = 1, - Delete = 2 - } - - [GenerateSerializer] - public sealed partial class JournaledStatePostCommitException : System.Exception - { - public JournaledStatePostCommitException(JournaledStateOperation operation, System.Exception innerException) { } - } - [GenerateSerializer] public sealed partial class JournaledStatePreCommitException : System.Exception { - public JournaledStatePreCommitException(JournaledStateOperation operation, System.Exception innerException) { } + public JournaledStatePreCommitException(System.Exception innerException) { } } public readonly ref partial struct JournalEntry @@ -828,24 +807,6 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. where TBufferWriter : System.Buffers.IBufferWriter { } } - [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] - [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] - public sealed partial class Codec_JournaledStatePostCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec - { - public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } - - public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePostCommitException instance) { } - - public global::Orleans.Journaling.JournaledStatePostCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } - - public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePostCommitException instance) - where TBufferWriter : System.Buffers.IBufferWriter { } - - public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePostCommitException value) - where TBufferWriter : System.Buffers.IBufferWriter { } - } - [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] @@ -871,14 +832,6 @@ public sealed partial class Copier_DurableTaskCompletionSourceState : global: { } - [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] - [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] - [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] - public sealed partial class Copier_JournaledStatePostCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier - { - public Copier_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } - } - [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index 5cb36fbffcf..c94bd6045e2 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -126,23 +126,6 @@ private void ReplayEncodedJournalData() private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter buffer, JournalStreamId streamId) : IJournaledStateManager { - private readonly List _hooks = []; - - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) - { - _hooks.Insert(index ?? _hooks.Count, hook); - return default; - } - - public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) - { - if (!_hooks.Remove(hook)) - { - throw new InvalidOperationException("The journal operation hook is not registered."); - } - return default; - } - public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(buffer.CreateJournalStreamWriter(streamId)); diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index 39e4213c5c1..7a9862c5cab 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,10 +727,6 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => - inner.AddHookAsync(hook, index, cancellationToken); - public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => - inner.RemoveHookAsync(hook, cancellationToken); public int DisposeCalls { get; private set; } public long PendingWriteByteCount => inner.PendingWriteByteCount; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); diff --git a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs index 4e975de5f74..10ac2a9537c 100644 --- a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs +++ b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs @@ -293,31 +293,27 @@ public async Task Recovery_EmptyJournalWithStaleMetadata_WritesConfiguredFormat( Assert.Contains("""[8,["set","alpha",1]]""", Encoding.UTF8.GetString(storage.Segments.Single()), StringComparison.Ordinal); } - [Theory] - [InlineData(OrleansBinaryJournalFormat.JournalFormatKey, JsonLinesJournalFormat.JournalFormatKey)] - [InlineData(JsonLinesJournalFormat.JournalFormatKey, OrleansBinaryJournalFormat.JournalFormatKey)] - public async Task Migration_WithUnregisteredRetiredState_PreservesStorage(string sourceFormat, string targetFormat) + [Fact] + public async Task Migration_WithUnregisteredRetiredState_ThrowsClearError() { - var storage = new VolatileJournalStorage(sourceFormat); - using var first = CreateFormatAwareTestSystem(storage, sourceFormat); - var dict = CreateFormatAwareDictionary(first, sourceFormat, "dict"); + var storage = new VolatileJournalStorage(OrleansBinaryJournalFormat.JournalFormatKey); + using var first = CreateFormatAwareTestSystem(storage, OrleansBinaryJournalFormat.JournalFormatKey); + var dict = CreateFormatAwareDictionary(first, OrleansBinaryJournalFormat.JournalFormatKey, "dict"); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); await first.Manager.WriteStateAsync(CancellationToken.None); - var originalBytes = Assert.Single(storage.Segments).ToArray(); - storage.SetConfiguredJournalFormatKey(targetFormat); - using var recovered = CreateFormatAwareTestSystem(storage, targetFormat); - var other = CreateFormatAwareDictionary(recovered, targetFormat, "other"); + storage.SetConfiguredJournalFormatKey(JsonLinesJournalFormat.JournalFormatKey); + using var recovered = CreateFormatAwareTestSystem(storage, JsonLinesJournalFormat.JournalFormatKey); + var other = CreateFormatAwareDictionary(recovered, JsonLinesJournalFormat.JournalFormatKey, "other"); await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); other.Add("beta", 2); var exception = await Assert.ThrowsAsync( () => recovered.Manager.WriteStateAsync(CancellationToken.None).AsTask()); - Assert.Contains("cannot append preserved entry", exception.Message, StringComparison.Ordinal); - Assert.Equal(sourceFormat, storage.StoredJournalFormatKey); - Assert.Equal(originalBytes, Assert.Single(storage.Segments)); + Assert.Contains("Cannot migrate journal", exception.Message, StringComparison.Ordinal); + Assert.Contains("not currently registered", exception.Message, StringComparison.Ordinal); } [Theory] diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index ede9edb2f56..26cf16aac46 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -193,23 +193,6 @@ public void Collections_UseDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { - private readonly List _hooks = []; - - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) - { - _hooks.Insert(index ?? _hooks.Count, hook); - return default; - } - - public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) - { - if (!_hooks.Remove(hook)) - { - throw new InvalidOperationException("The journal operation hook is not registered."); - } - return default; - } - public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 245561f8c50..e1cc3708672 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -92,23 +92,6 @@ public void Add_UsesDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { - private readonly List _hooks = []; - - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) - { - _hooks.Insert(index ?? _hooks.Count, hook); - return default; - } - - public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) - { - if (!_hooks.Remove(hook)) - { - throw new InvalidOperationException("The journal operation hook is not registered."); - } - return default; - } - public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs index e8355089172..b145f0f7417 100644 --- a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs @@ -631,7 +631,7 @@ private static void AssertIncompatible( private static void AssertAdmissionRejected(Action action) { var error = Assert.Throws(action); - Assert.Equal("Journal registrations cannot be changed after journaled state manager initialization has begun.", error.Message); + Assert.Equal("New states cannot be registered after journaled state manager initialization has begun.", error.Message); } private static void AssertFenced(InvalidOperationException error, IOException cause) diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index dd7945ec7cb..e9822dd9557 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,10 +746,6 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => - inner.AddHookAsync(hook, index, cancellationToken); - public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => - inner.RemoveHookAsync(hook, cancellationToken); public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs deleted file mode 100644 index 8833f87cb85..00000000000 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ /dev/null @@ -1,1566 +0,0 @@ -using System.Buffers; -using System.Collections.Concurrent; -using System.Diagnostics; -using System.Reflection; -using System.Threading.Channels; -using Microsoft.Extensions.DependencyInjection; -using NSubstitute; -using Orleans.Diagnostics; -using Orleans.Serialization; -using Xunit; - -namespace Orleans.Journaling.Tests; - -public partial class StateManagerTests -{ - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorageOutcome(bool snapshot, bool storageFails) - { - using var parent = new Activity("journal queue test").SetIdFormat(ActivityIdFormat.W3C).Start(); - var stopped = new ConcurrentBag(); - using var listener = new ActivityListener - { - ShouldListenTo = source => source.Name == ActivitySources.StorageActivitySourceName, - Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, - ActivityStopped = activity => - { - if (activity.ParentSpanId == parent.SpanId) - { - stopped.Add(activity); - } - } - }; - ActivitySource.AddActivityListener(listener); - var failure = new IOException("Storage failed."); - var storage = new CapturingStorage - { - IsCompactionRequested = snapshot, - NextAppendException = storageFails && !snapshot ? failure : null, - NextReplaceException = storageFails && snapshot ? failure : null - }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - if (storageFails) - { - Assert.Same(failure, await Assert.ThrowsAsync(() => - manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); - } - else - { - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - } - - await manager.DisposeAsync(); - var activity = Assert.Single(stopped); - var operation = snapshot ? "snapshot" : "append"; - Assert.Equal($"journal {operation}", activity.OperationName); - Assert.Equal(operation, activity.GetTagItem(ActivityTagKeys.JournalStorageOperation)); - Assert.Equal(storageFails ? ActivityStatusCode.Error : ActivityStatusCode.Ok, activity.Status); - Assert.Equal(storageFails ? failure.Message : null, activity.StatusDescription); - } - - [Fact] - public async Task Hooks_IdleRegistrationAppliesSynchronously() - { - await using var manager = CreateTestSystem().Manager; - _ = new DurableValue("state", manager, CreateValueCodec()); - var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; - Assert.Null(field.GetValue(manager)); - - var before = 0; - var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; - Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - await Assert.ThrowsAsync(() => manager.AddHookAsync(null!, cancellationToken: TestContext.Current.CancellationToken).AsTask()); - await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(0, before); - Assert.True(manager.AddHookAsync(hook, index: 0, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, before); - Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, before); - - await using var withoutHooks = CreateTestSystem().Manager; - await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); - await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Null(field.GetValue(withoutHooks)); - } - - [Fact] - public async Task Hooks_ShutdownRejectsRegistrationAndRemoval() - { - var manager = CreateTestSystem().Manager; - var hook = new DelegateTestHook(); - await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); - - await manager.DisposeAsync(); - - await Assert.ThrowsAsync(() => manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); - await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); - } - - [Fact] - public async Task Hooks_RegistrationDuringRecoveryAppliesBeforeLaterPersistence() - { - var entered = NewSignal(); - var release = NewSignal(); - var storage = Substitute.For(); - storage.ReadAsync(Arg.Any(), Arg.Any()) - .Returns(call => ReadAsync(call.Arg(), call.Arg())); - await using var manager = CreateTestSystem(storage).Manager; - var before = 0; - var after = 0; - var hook = new DelegateTestHook - { - BeforeOperation = (_, _) => before++, - AfterOperation = (_, _) => after++ - }; - var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); - var registration = manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - Assert.False(initialization.IsCompleted); - Assert.False(registration.IsCompleted); - release.TrySetResult(); - await WaitFor(Task.WhenAll(initialization, registration)); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, before); - Assert.Equal(1, after); - } - finally - { - release.TrySetResult(); - } - - async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) - { - entered.TrySetResult(); - await release.Task.WaitAsync(token); - consumer.Complete(metadata: null); - } - } - - [Fact] - public async Task Hooks_QueuedRegistrationReportsRecoveryFailureAndIdleRegistrationDoesNotRetryRecovery() - { - var entered = NewSignal(); - var release = NewSignal(); - var storage = Substitute.For(); - var failure = new IOException("Recovery failed."); - var fail = true; - var reads = 0; - storage.ReadAsync(Arg.Any(), Arg.Any()) - .Returns(call => ReadAsync(call.Arg(), call.Arg())); - await using var manager = CreateTestSystem(storage).Manager; - var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); - var failedBefore = 0; - var failedRegistration = manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => failedBefore++ - }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - release.TrySetResult(); - Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); - Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(failedRegistration))); - var before = 0; - var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; - Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - Assert.Equal(1, reads); - fail = false; - await manager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, reads); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(0, failedBefore); - Assert.Equal(1, before); - } - finally - { - release.TrySetResult(); - } - - async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) - { - reads++; - entered.TrySetResult(); - await release.Task.WaitAsync(token); - if (fail) - { - throw failure; - } - consumer.Complete(metadata: null); - } - } - - [Theory] - [InlineData("before")] - [InlineData("storage")] - [InlineData("after")] - public async Task Hooks_BusyMutationWaitsForCompleteOperationAndSurvivesCallerCancellation(string phase) - { - var storage = new CapturingStorage { BlockNextAppend = phase == "storage" }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - var entered = NewSignal(); - var release = NewSignal(); - var oldBefore = 0; - var oldAfter = 0; - var newBefore = 0; - var newAfter = 0; - var original = new DelegateTestHook - { - BeforeOperationAsync = async (_, _) => - { - oldBefore++; - if (phase == "before") - { - entered.TrySetResult(); - await release.Task; - } - }, - AfterOperationAsync = async (_, _) => - { - oldAfter++; - if (phase == "after") - { - entered.TrySetResult(); - await release.Task; - } - } - }; - var replacement = new DelegateTestHook - { - BeforeOperation = (_, _) => newBefore++, - AfterOperation = (_, _) => newAfter++ - }; - await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(phase == "storage" ? storage.BlockedAppendStarted.Task : entered.Task); - using var caller = new CancellationTokenSource(); - var added = manager.AddHookAsync(replacement, index: 0, caller.Token).AsTask(); - var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - state.Value = 2; - var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - Assert.False(added.IsCompleted); - Assert.False(removed.IsCompleted); - Assert.Equal(0, newBefore); - Assert.Equal(0, newAfter); - caller.Cancel(); - await Assert.ThrowsAnyAsync(() => added); - - release.TrySetResult(); - storage.ReleaseAppend.TrySetResult(); - await WaitFor(Task.WhenAll(first, removed, next)); - Assert.Equal(1, oldBefore); - Assert.Equal(1, oldAfter); - Assert.Equal(1, newBefore); - Assert.Equal(1, newAfter); - Assert.Equal(2, state.Value); - } - finally - { - release.TrySetResult(); - storage.ReleaseAppend.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_QueuedMutationSeparatesWriteCoalescingGroups() - { - var storage = new CapturingStorage { BlockNextAppend = true }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new LifecycleState(); - manager.RegisterStateMachine("state", state); - var oldBefore = 0; - var oldAfter = 0; - var newBefore = 0; - var newAfter = 0; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => oldBefore++, - AfterOperation = (_, _) => oldAfter++ - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(storage.BlockedAppendStarted.Task); - var beforeChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - var change = manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => newBefore++, - AfterOperation = (_, _) => newAfter++ - }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - var afterChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - var coalesced = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - storage.ReleaseAppend.TrySetResult(); - await WaitFor(Task.WhenAll(first, beforeChange, change, afterChange, coalesced)); - Assert.Equal(3, state.CaptureCount); - Assert.Equal(3, storage.Appends.Count); - Assert.Equal(3, oldBefore); - Assert.Equal(3, oldAfter); - Assert.Equal(1, newBefore); - Assert.Equal(1, newAfter); - } - finally - { - storage.ReleaseAppend.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_ResetCanQueueRegistrationWithoutChangingCurrentAfterHooks() - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new LifecycleState(); - Task? registration = null; - var newBefore = 0; - var newAfter = 0; - state.ResetAction = () => - { - if (state.ResetCount == 2) - { - registration = manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => newBefore++, - AfterOperation = (_, _) => newAfter++ - }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - } - }; - manager.RegisterStateMachine("state", state); - await manager.AddHookAsync(new DelegateTestHook - { - AfterOperation = (operation, _) => - { - if (operation == JournaledStateOperation.Delete) - { - Assert.NotNull(registration); - Assert.False(registration.IsCompleted); - Assert.Equal(0, newBefore); - Assert.Equal(0, newAfter); - } - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.DeleteStateAsync(TestContext.Current.CancellationToken); - Assert.NotNull(registration); - await WaitFor(registration); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, newBefore); - Assert.Equal(1, newAfter); - } - - [Fact] - public async Task Hooks_FailedQueuedMutationReportsErrorAndLeavesOwnerUsable() - { - var storage = new CapturingStorage { BlockNextAppend = true }; - await using var manager = CreateTestSystem(storage).Manager; - manager.RegisterStateMachine("state", new LifecycleState()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(storage.BlockedAppendStarted.Task); - var invalid = manager.AddHookAsync(new DelegateTestHook(), index: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - var before = 0; - var valid = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; - var accepted = manager.AddHookAsync(valid, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - storage.ReleaseAppend.TrySetResult(); - await Assert.ThrowsAsync(() => WaitFor(invalid)); - await WaitFor(Task.WhenAll(first, accepted)); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, before); - } - finally - { - storage.ReleaseAppend.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_StorageFailureFaultsQueuedRegistrationWithOriginalCause() - { - var failure = new IOException("Storage failed."); - var storage = new CapturingStorage { BlockNextAppend = true, NextAppendException = failure }; - await using var manager = CreateTestSystem(storage).Manager; - manager.RegisterStateMachine("state", new LifecycleState()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(storage.BlockedAppendStarted.Task); - var registration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); - storage.ReleaseAppend.TrySetResult(); - Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(write))); - Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(registration))); - var fenced = await Assert.ThrowsAsync(() => - manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask()); - Assert.Same(failure, fenced.InnerException); - } - finally - { - storage.ReleaseAppend.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_ShutdownCancelsQueuedMutationWithoutChangingCommittedOperation() - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - manager.RegisterStateMachine("state", new LifecycleState()); - var entered = NewSignal(); - var release = NewSignal(); - var original = new DelegateTestHook - { - AfterOperationAsync = async (_, _) => - { - entered.TrySetResult(); - await release.Task; - } - }; - await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - var added = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); - var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - var shutdown = manager.DisposeAsync().AsTask(); - Assert.False(shutdown.IsCompleted); - release.TrySetResult(); - await WaitFor(Task.WhenAll(write, shutdown)); - await Assert.ThrowsAnyAsync(() => WaitFor(added)); - await Assert.ThrowsAnyAsync(() => WaitFor(removed)); - Assert.Single(storage.Appends); - } - finally - { - release.TrySetResult(); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bool snapshot) - { - var storage = new CapturingStorage { IsCompactionRequested = snapshot, BlockNextAppend = !snapshot, BlockNextReplace = snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - List events = []; - manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; - CancellationToken ownedToken = default; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (operation, token) => - { - Assert.Equal(expected, operation); - Assert.True(token.CanBeCanceled); - ownedToken = token; - events.Add("before-sync"); - state["hook"] = 2; - }, - BeforeOperationAsync = async (operation, token) => - { - Assert.Equal(expected, operation); - Assert.Equal(ownedToken, token); - await Task.Yield(); - events.Add("before-async"); - }, - AfterOperation = (operation, token) => - { - Assert.Equal(expected, operation); - Assert.Equal(ownedToken, token); - Assert.Single(snapshot ? storage.Replaces : storage.Appends); - events.Add("after-sync"); - }, - AfterOperationAsync = async (_, token) => - { - Assert.Equal(ownedToken, token); - await Task.Yield(); - events.Add("after-async"); - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await (snapshot ? storage.ReplaceEntered : storage.BlockedAppendStarted).Task - .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(new[] { "before-sync", "before-async", "capture" }, events); - Assert.False(write.IsCompleted); - (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); - await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(new[] { "before-sync", "before-async", "capture", "ack", "after-sync", "after-async" }, events); - } - finally - { - (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); - } - - await using var replayManager = CreateTestSystem(storage).Manager; - var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); - await replayManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, recovered.Count); - Assert.Equal(1, recovered["business"]); - Assert.Equal(2, recovered["hook"]); - } - - [Theory] - [InlineData(JournaledStateOperation.Write)] - [InlineData(JournaledStateOperation.Snapshot)] - [InlineData(JournaledStateOperation.Delete)] - public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledStateOperation operation) - { - var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - var capture = new AlwaysWritingState(); - manager.RegisterStateMachine("capture", capture); - var failure = new IOException("Prerequisite failed."); - var fail = true; - var after = 0; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => - { - if (fail) - { - throw failure; - } - }, - AfterOperation = (_, _) => after++ - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; - var pendingBytes = manager.PendingWriteByteCount; - var caught = await Assert.ThrowsAsync(() => InvokeAsync()); - Assert.Same(failure, caught.InnerException); - Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); - Assert.Equal(1, state["business"]); - Assert.Equal(pendingBytes, manager.PendingWriteByteCount); - Assert.Equal(0, capture.AppendEntriesCount); - Assert.Equal(0, capture.WriteCompletedCount); - Assert.Empty(storage.Appends); - Assert.Empty(storage.Replaces); - Assert.Equal(0, storage.DeleteCount); - Assert.Equal(0, after); - storage.IsCompactionRequested = false; - fail = false; - await InvokeAsync(); - Assert.Equal(1, after); - Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); - Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); - Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); - Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); - - Task InvokeAsync() => (operation == JournaledStateOperation.Delete - ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) - : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask(); - } - - [Theory] - [InlineData(JournaledStateOperation.Write)] - [InlineData(JournaledStateOperation.Snapshot)] - [InlineData(JournaledStateOperation.Delete)] - public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHooks(JournaledStateOperation operation) - { - var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - List events = []; - manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - var first = new IOException("First cleanup failed."); - var second = new IOException("Second cleanup failed."); - var fail = true; - await manager.AddHookAsync(new DelegateTestHook - { - AfterOperation = (_, _) => - { - if (fail) - { - throw first; - } - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook - { - AfterOperationAsync = async (_, _) => - { - await Task.Yield(); - events.Add("second"); - if (fail) - { - throw second; - } - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; - var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete - ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) - : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); - Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); - Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); - Assert.Equal(operation == JournaledStateOperation.Delete - ? new[] { "second", "last" } - : new[] { "capture", "ack", "second", "last" }, events); - Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); - Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); - fail = false; - - await using var replayManager = CreateTestSystem(storage).Manager; - var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); - await replayManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, recovered.Count); - if (operation != JournaledStateOperation.Delete) - { - Assert.Equal(1, recovered["business"]); - } - - state["later"] = 2; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, state["later"]); - } - - [Fact] - public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() - { - var failure = new IOException("Storage failed."); - var storage = new CapturingStorage { NextAppendException = failure }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - var after = 0; - await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; - Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); - Assert.Equal(0, after); - var fenced = await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); - Assert.Same(failure, fenced.InnerException); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinaryHook(bool snapshot) - { - var storage = new CapturingStorage { IsCompactionRequested = snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - List events = []; - manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - await manager.AddHookAsync(new DelegateTestHook( - (_, _) => - { - Assert.Equal(3, state["late"]); - state["owner"] = 2; - events.Add("capture-prerequisite"); - return default; - }, - (_, _) => events.Add("capture-after")), cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperationAsync = async (_, token) => - { - events.Add("ordinary-before"); - entered.TrySetResult(); - await release.Task.WaitAsync(token); - events.Add("ordinary-complete"); - }, - AfterOperation = (_, _) => events.Add("ordinary-after") - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(new[] { "ordinary-before" }, events); - Assert.Empty(storage.Appends); - Assert.Empty(storage.Replaces); - state["late"] = 3; - release.TrySetResult(); - await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(new[] { "ordinary-before", "ordinary-complete", "capture-prerequisite", "capture", "ack", "capture-after", "ordinary-after" }, events); - } - finally - { - release.TrySetResult(); - } - - await using var recoveredManager = CreateTestSystem(storage).Manager; - var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); - await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(3, recovered.Count); - Assert.Equal(1, recovered["business"]); - Assert.Equal(2, recovered["owner"]); - Assert.Equal(3, recovered["late"]); - } - - [Fact] - public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - CancellationToken ownedToken = default; - Func before = async (_, token) => - { - ownedToken = token; - entered.TrySetResult(); - await release.Task.WaitAsync(token); - state["scheduled"] = 2; - }; - Action after = (_, token) => - { - Assert.Equal(ownedToken, token); - Assert.False(token.IsCancellationRequested); - Assert.Single(storage.Appends); - completed.TrySetResult(); - }; - await manager.AddHookAsync(new DelegateTestHook(before, after), cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - using var caller = new CancellationTokenSource(); - state["business"] = 1; - var write = manager.WriteStateAsync(caller.Token).AsTask(); - try - { - await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.NotEqual(caller.Token, ownedToken); - state["late"] = 3; - caller.Cancel(); - await Assert.ThrowsAnyAsync(() => write); - Assert.False(ownedToken.IsCancellationRequested); - Assert.Empty(storage.Appends); - Assert.False(completed.Task.IsCompleted); - release.TrySetResult(); - await completed.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(2, state["scheduled"]); - await using var recoveredManager = CreateTestSystem(storage).Manager; - var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); - await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(3, recovered.Count); - Assert.Equal(1, recovered["business"]); - Assert.Equal(2, recovered["scheduled"]); - Assert.Equal(3, recovered["late"]); - } - finally - { - release.TrySetResult(); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_SynchronousFinalPrerequisiteAndCaptureShareContinuationAfterAsyncEarlierHook(bool snapshot) - { - var scheduler = new QueuedHookTaskScheduler(); - await scheduler.Run(async () => - { - var storage = new CapturingStorage { IsCompactionRequested = snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - List events = []; - List prepared = []; - manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - var queueMutation = true; - Task? mutation = null; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => - { - events.Add("final-before"); - prepared.AddRange(state.Keys); - if (queueMutation) - { - Assert.Empty(state); - queueMutation = false; - mutation = scheduler.Queue(() => - { - state["intent"] = 1; - events.Add("queued-mutation"); - }); - } - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperationAsync = async (_, _) => - { - events.Add("ordinary-before"); - await Task.Yield(); - events.Add("ordinary-complete"); - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - - Assert.NotNull(mutation); - await mutation; - Assert.Empty(prepared); - Assert.Equal(1, state["intent"]); - await using (var recoveredManager = CreateTestSystem(storage).Manager) - { - var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); - await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Empty(recovered); - } - - Assert.Equal( - ["ordinary-before", "ordinary-complete", "final-before", "capture", "ack", "queued-mutation"], - events); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(["intent"], prepared); - await using var replayManager = CreateTestSystem(storage).Manager; - var replayed = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); - await replayManager.InitializeAsync(TestContext.Current.CancellationToken); - var intent = Assert.Single(replayed); - Assert.Equal("intent", intent.Key); - Assert.Equal(1, intent.Value); - }); - } - - [Fact] - public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrder() - { - await using var manager = CreateTestSystem().Manager; - List events = []; - manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => events.Add("second-before"), - AfterOperation = (_, _) => events.Add("second-after") - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => events.Add("third-before"), - AfterOperation = (_, _) => events.Add("third-after") - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => events.Add("first-before"), - AfterOperation = (_, _) => events.Add("first-after") - }, index: 0, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal( - ["third-before", "second-before", "first-before", "capture", "ack", "first-after", "second-after", "third-after"], - events); - } - - [Fact] - public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - var deleting = false; - var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var before = 0; - var after = 0; - await manager.AddHookAsync(new DelegateTestHook( - async (operation, token) => - { - before++; - if (!deleting) - { - Assert.Equal(JournaledStateOperation.Write, operation); - return; - } - - Assert.Equal(JournaledStateOperation.Delete, operation); - Assert.Equal(1, state.Value); - entered.TrySetResult(); - await release.Task.WaitAsync(token); - }, - (operation, _) => - { - if (!deleting) - { - return; - } - - Assert.Equal(JournaledStateOperation.Delete, operation); - Assert.Equal(0, state.Value); - Assert.Equal(1, storage.DeleteCount); - after++; - }), cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - deleting = true; - var deletion = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(0, storage.DeleteCount); - Assert.Equal(0, after); - Assert.False(deletion.IsCompleted); - release.TrySetResult(); - await deletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(1, after); - deleting = false; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(3, before); - } - finally - { - release.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() - { - var storage = new CapturingStorage(); - var manager = CreateTestSystem(storage).Manager; - var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var drained = false; - var failure = new IOException("Cancellation callback failed."); - var hook = new DelegateTestHook - { - BeforeOperationAsync = async (_, token) => - { - using var registration = token.Register(() => throw failure); - entered.TrySetResult(); - await release.Task; - drained = true; - } - }; - await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - var shutdown = manager.DisposeAsync().AsTask(); - var concurrentShutdown = manager.DisposeAsync().AsTask(); - Assert.Same(shutdown, concurrentShutdown); - Assert.False(shutdown.IsCompleted); - Assert.False(drained); - release.TrySetResult(); - var caught = await Assert.ThrowsAsync(() => - shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - Assert.Same(failure, Assert.Single(caught.InnerExceptions)); - await Assert.ThrowsAnyAsync(() => write); - await Assert.ThrowsAnyAsync(() => queued); - Assert.True(drained); - Assert.Empty(storage.Appends); - await Assert.ThrowsAsync(() => - manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); - } - finally - { - release.TrySetResult(); - await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDrains(bool after) - { - var storage = new CapturingStorage(); - var manager = CreateTestSystem(storage).Manager; - var state = new LifecycleState(); - manager.RegisterStateMachine("state", state); - var entered = NewSignal(); - var release = NewSignal(); - Func callback = async (_, _) => - { - entered.TrySetResult(); - await release.Task; - }; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperationAsync = after ? null : callback, - AfterOperationAsync = after ? callback : null - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - var initializing = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); - var deleting = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); - var shutdown = manager.DisposeAsync().AsTask(); - Assert.False(shutdown.IsCompleted); - Assert.False(initializing.IsCompleted); - Assert.False(deleting.IsCompleted); - release.TrySetResult(); - await WaitFor(shutdown); - if (after) - { - await WaitFor(write); - } - else - { - await Assert.ThrowsAnyAsync(() => WaitFor(write)); - } - - await Assert.ThrowsAnyAsync(() => WaitFor(initializing)); - await Assert.ThrowsAnyAsync(() => WaitFor(deleting)); - Assert.True(initializing.IsCanceled); - Assert.True(deleting.IsCanceled); - Assert.Equal(after ? 1 : 0, state.CaptureCount); - Assert.Equal(after ? 1 : 0, state.WriteCompletedCount); - Assert.Equal(after ? 1 : 0, storage.Appends.Count); - Assert.Equal(0, storage.DeleteCount); - } - finally - { - release.TrySetResult(); - await manager.DisposeAsync(); - } - } - - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesItsOutcome(bool snapshot, bool storageFails) - { - var entered = NewSignal(); - var release = NewSignal(); - var cancellationFailure = new IOException("Cancellation callback failed."); - var storageFailure = new IOException("Actual storage failure."); - var completed = false; - var storage = Substitute.For(); - storage.IsCompactionRequested.Returns(snapshot); - storage.ReadAsync(Arg.Any(), Arg.Any()).Returns(call => - { - call.Arg().Complete(metadata: null); - return ValueTask.CompletedTask; - }); - storage.AppendAsync(Arg.Any>(), Arg.Any()) - .Returns(call => WriteAsync(call.Arg>(), call.Arg())); - storage.ReplaceAsync(Arg.Any>(), Arg.Any()) - .Returns(call => WriteAsync(call.Arg>(), call.Arg())); - var manager = CreateTestSystem(storage).Manager; - var state = new LifecycleState(); - manager.RegisterStateMachine("state", state); - CancellationTokenRegistration registration = default; - var after = 0; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), - AfterOperation = (_, _) => after++ - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - var shutdown = manager.DisposeAsync().AsTask(); - Assert.Same(shutdown, manager.DisposeAsync().AsTask()); - Assert.False(shutdown.IsCompleted); - Assert.False(completed); - Assert.Equal(0, state.WriteCompletedCount); - Assert.Equal(0, after); - release.TrySetResult(); - var caught = await Assert.ThrowsAsync(() => WaitFor(shutdown)); - Assert.Same(cancellationFailure, Assert.Single(caught.InnerExceptions)); - Assert.True(completed); - if (storageFails) - { - Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(write))); - Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(queued))); - } - else - { - await WaitFor(write); - await Assert.ThrowsAnyAsync(() => WaitFor(queued)); - } - - Assert.Equal(storageFails ? 0 : 1, state.WriteCompletedCount); - Assert.Equal(storageFails ? 0 : 1, after); - } - finally - { - release.TrySetResult(); - registration.Dispose(); - await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); - } - - async ValueTask WriteAsync(ReadOnlySequence bytes, CancellationToken token) - { - var original = bytes.ToArray(); - entered.TrySetResult(); - await release.Task; - Assert.True(token.IsCancellationRequested); - Assert.Equal(original, bytes.ToArray()); - completed = true; - if (storageFails) - { - throw storageFailure; - } - } - } - - [Fact] - public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() - { - var storage = new CapturingStorage(); - var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - var entered = NewSignal(); - var release = NewSignal(); - Task? reenteredDisposal = null; - var callbackCount = 0; - await manager.AddHookAsync(new DelegateTestHook - { - AfterOperationAsync = async (_, token) => - { - using var registration = token.Register(() => - { - callbackCount++; - reenteredDisposal = manager.DisposeAsync().AsTask(); - }); - entered.TrySetResult(); - await release.Task; - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - var disposal = manager.DisposeAsync().AsTask(); - Assert.Equal(1, callbackCount); - Assert.Same(disposal, reenteredDisposal); - Assert.False(disposal.IsCompleted); - release.TrySetResult(); - await WaitFor(Task.WhenAll(write, disposal)); - Assert.Same(disposal, manager.DisposeAsync().AsTask()); - Assert.Single(storage.Appends); - } - finally - { - release.TrySetResult(); - await manager.DisposeAsync(); - } - } - - [Fact] - public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() - { - var storage = new CapturingStorage(); - var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var first = new IOException("First cleanup failed."); - var second = new IOException("Second cleanup failed."); - List events = []; - await manager.AddHookAsync(new DelegateTestHook - { - AfterOperationAsync = async (_, token) => - { - Assert.Single(storage.Appends); - entered.TrySetResult(); - await release.Task; - Assert.True(token.IsCancellationRequested); - events.Add("first"); - throw first; - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => throw second }, cancellationToken: TestContext.Current.CancellationToken); - await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - var shutdown = manager.DisposeAsync().AsTask(); - var concurrentShutdown = manager.DisposeAsync().AsTask(); - Assert.Same(shutdown, concurrentShutdown); - Assert.False(shutdown.IsCompleted); - release.TrySetResult(); - var caught = await Assert.ThrowsAsync(() => - write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); - await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(new[] { "first", "last" }, events); - Assert.Single(storage.Appends); - } - finally - { - release.TrySetResult(); - await manager.DisposeAsync(); - } - } - - [Fact] - public async Task Hooks_QueuedWritesCoalesceAndRetainSnapshotRequirement() - { - var context = new QueuedSynchronizationContext(); - await context.Run(async () => - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - List before = []; - List after = []; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (operation, _) => before.Add(operation), - AfterOperation = (operation, _) => after.Add(operation) - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var append = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - storage.IsCompactionRequested = true; - state.Value = 2; - var snapshot = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - storage.IsCompactionRequested = false; - state.Value = 3; - var anotherAppend = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - - Assert.Empty(storage.Appends); - Assert.Empty(storage.Replaces); - Assert.Empty(before); - await Task.WhenAll(append, snapshot, anotherAppend); - Assert.Equal([JournaledStateOperation.Snapshot], before); - Assert.Equal([JournaledStateOperation.Snapshot], after); - Assert.Empty(storage.Appends); - Assert.Single(storage.Replaces); - - await using var recoveredManager = CreateTestSystem(storage).Manager; - var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); - await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(3, recovered.Value); - }); - } - - [Fact] - public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() - { - var storage = new CapturingStorage { IsCompactionRequested = true }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - var entered = NewSignal(); - var release = NewSignal(); - var failure = new IOException("Snapshot prerequisite failed."); - List before = []; - List after = []; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperationAsync = async (operation, _) => - { - before.Add(operation); - if (before.Count == 1) - { - entered.TrySetResult(); - await release.Task; - throw failure; - } - }, - AfterOperation = (operation, _) => after.Add(operation) - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - storage.IsCompactionRequested = false; - state.Value = 2; - var retry = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - Assert.Empty(storage.Appends); - Assert.Empty(storage.Replaces); - release.TrySetResult(); - var caught = await Assert.ThrowsAsync(() => WaitFor(first)); - Assert.Same(failure, caught.InnerException); - await WaitFor(retry); - Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); - Assert.Equal([JournaledStateOperation.Snapshot], after); - Assert.Empty(storage.Appends); - Assert.Single(storage.Replaces); - await using var recoveredManager = CreateTestSystem(storage).Manager; - var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); - await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, recovered.Value); - } - finally - { - release.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() - { - var storage = new CapturingStorage { IsCompactionRequested = true, BlockNextReplace = true }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - List before = []; - List after = []; - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (operation, _) => before.Add(operation), - AfterOperation = (operation, _) => after.Add(operation) - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(storage.ReplaceEntered.Task); - state.Value = 2; - var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - storage.IsCompactionRequested = false; - Assert.Empty(storage.Replaces); - Assert.Equal([JournaledStateOperation.Snapshot], before); - Assert.Empty(after); - storage.ReleaseReplace.TrySetResult(); - await WaitFor(Task.WhenAll(first, second)); - Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); - Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], after); - Assert.Empty(storage.Appends); - Assert.Equal(2, storage.Replaces.Count); - await using var recoveredManager = CreateTestSystem(storage).Manager; - var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); - await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, recovered.Value); - } - finally - { - storage.ReleaseReplace.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() - { - var storage = new CapturingStorage { BlockNextAppend = true }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - var before = 0; - var after = 0; - await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["first"] = 1; - var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await storage.BlockedAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(1, before); - Assert.Equal(0, after); - state["second"] = 2; - var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - state["third"] = 3; - var third = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - storage.ReleaseAppend.TrySetResult(); - await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Equal(2, before); - Assert.Equal(2, after); - Assert.Equal(2, storage.Appends.Count); - } - finally - { - storage.ReleaseAppend.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - var entered = NewSignal(); - var release = NewSignal(); - var after = 0; - await manager.AddHookAsync(new DelegateTestHook - { - AfterOperationAsync = async (_, _) => - { - if (++after == 1) - { - entered.TrySetResult(); - await release.Task; - } - } - }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["first"] = 1; - var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await WaitFor(entered.Task); - Assert.Single(storage.Appends); - Assert.False(first.IsCompleted); - state["later"] = 2; - var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - Assert.False(next.IsCompleted); - - await using (var beforeNext = CreateTestSystem(storage).Manager) - { - var recovered = new DurableDictionary("state", beforeNext, CreateDictionaryCodec()); - await beforeNext.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, Assert.Single(recovered).Value); - } - - release.TrySetResult(); - await WaitFor(Task.WhenAll(first, next)); - Assert.Equal(2, after); - Assert.Equal(2, storage.Appends.Count); - await using var afterNext = CreateTestSystem(storage).Manager; - var persisted = new DurableDictionary("state", afterNext, CreateDictionaryCodec()); - await afterNext.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, persisted.Count); - Assert.Equal(1, persisted["first"]); - Assert.Equal(2, persisted["later"]); - } - finally - { - release.TrySetResult(); - } - } - - [Fact] - public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - var before = 0; - var after = 0; - await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, before); - Assert.Equal(1, after); - before = 0; - after = 0; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, before); - Assert.Equal(1, after); - Assert.Single(storage.Appends); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) - { - Exception original = after - ? new JournaledStatePostCommitException(JournaledStateOperation.Delete, new IOException("Cleanup failed.")) - : new JournaledStatePreCommitException(JournaledStateOperation.Snapshot, new IOException("Scheduling failed.")); - var serializer = ServiceProvider.GetRequiredService(); - var copy = serializer.Deserialize(serializer.SerializeToArray(original)); - if (after) - { - Assert.IsType(copy); - } - else - { - Assert.IsType(copy); - } - - Assert.Equal(original.InnerException!.Message, Assert.IsType(copy.InnerException).Message); - Assert.Equal(original.Message, copy.Message); - } - - private sealed class QueuedHookTaskScheduler : TaskScheduler - { - private readonly Channel _queue = Channel.CreateUnbounded(); - - public override int MaximumConcurrencyLevel => 1; - protected override IEnumerable? GetScheduledTasks() => null; - protected override void QueueTask(Task task) => Assert.True(_queue.Writer.TryWrite(task)); - protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; - - public Task Queue(Action action) => - Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this); - - public async Task Run(Func action) - { - var task = Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this).Unwrap(); - while (!task.IsCompleted) - { - var next = await _queue.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); - var previous = SynchronizationContext.Current; - SynchronizationContext.SetSynchronizationContext(null); - try - { - Assert.True(TryExecuteTask(next)); - } - finally - { - SynchronizationContext.SetSynchronizationContext(previous); - } - } - - await task; - } - } - - private sealed class DelegateTestHook( - Func? before = null, - Action? after = null) : IJournaledStateHook - { - public Action? BeforeOperation { get; init; } - public Func? BeforeOperationAsync { get; init; } = before; - public Action? AfterOperation { get; init; } = after; - public Func? AfterOperationAsync { get; init; } - - ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - BeforeOperation?.Invoke(operation, cancellationToken); - return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; - } - - ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - AfterOperation?.Invoke(operation, cancellationToken); - return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; - } - } - - private sealed class HookAcknowledgementState(List events) : IStateMachine - { - public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } - public void Reset(JournalStreamWriter writer) { } - public void WritePendingEntries(JournalStreamWriter writer) => events.Add("capture"); - public void WriteSnapshot(JournalStreamWriter writer) => events.Add("capture"); - public void OnWriteCompleted() => events.Add("ack"); - } -} diff --git a/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs index 742bc1a2e77..fed449385bd 100644 --- a/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs @@ -1,3 +1,6 @@ +using System.Threading.Channels; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; using Xunit; namespace Orleans.Journaling.Tests; @@ -7,27 +10,22 @@ public partial class StateManagerTests [Theory] [InlineData(false)] [InlineData(true)] - public async Task Preparation_RunsAfterHooksBeforeCaptureAndAcknowledgement(bool snapshot) + public async Task Preparation_RunsBeforeCaptureAndAcknowledgement(bool snapshot) { var storage = new CapturingStorage { IsCompactionRequested = snapshot }; await using var manager = CreateTestSystem(storage).Manager; List events = []; var state = new PreparingValue("state", manager, CreateValueCodec(), events); - await manager.AddHookAsync(new DelegateTestHook - { - BeforeOperation = (_, _) => { events.Add("before"); state["intent"] = 17; }, - AfterOperation = (_, _) => events.Add("after") - }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["intent"] = 17; await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(["before", "prepare", "capture", "ack", "after"], events); + Assert.Equal(["prepare", "capture", "ack"], events); Assert.Equal(17, state.CapturedValue); Assert.Equal(1, state.WriteCompletedCount); Assert.Single(snapshot ? storage.Replaces : storage.Appends); } - [Theory] [InlineData(false)] [InlineData(true)] @@ -84,7 +82,7 @@ public async Task Preparation_LaterAsynchronousStateRechecksEarlierInterleavedCh [InlineData(true)] public async Task Preparation_SynchronousReadyPassAndCaptureShareOneTurn(bool snapshot) { - var scheduler = new QueuedHookTaskScheduler(); + var scheduler = new QueuedTaskScheduler(); await scheduler.Run(async () => { var storage = new CapturingStorage { IsCompactionRequested = snapshot }; @@ -222,6 +220,51 @@ public async Task Preparation_ShutdownDrainsPreparationAndPreventsCapture() } } + [Fact] + public void PreCommitFailure_RoundTripsWithOriginalCause() + { + var original = new JournaledStatePreCommitException(new IOException("Scheduling failed.")); + var serializer = ServiceProvider.GetRequiredService(); + + var copy = Assert.IsType(serializer.Deserialize(serializer.SerializeToArray(original))); + + Assert.Equal(original.Message, copy.Message); + Assert.Equal("Scheduling failed.", Assert.IsType(copy.InnerException).Message); + } + + private sealed class QueuedTaskScheduler : TaskScheduler + { + private readonly Channel _queue = Channel.CreateUnbounded(); + + public override int MaximumConcurrencyLevel => 1; + protected override IEnumerable? GetScheduledTasks() => null; + protected override void QueueTask(Task task) => Assert.True(_queue.Writer.TryWrite(task)); + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; + + public Task Queue(Action action) => + Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this); + + public async Task Run(Func action) + { + var task = Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this).Unwrap(); + while (!task.IsCompleted) + { + var next = await _queue.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(null); + try + { + Assert.True(TryExecuteTask(next)); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + } + } + + await task; + } + } private sealed class PreparingValue : IStateMachine, IDurableValueCommandHandler { private readonly IDurableValueCommandCodec _codec; diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs index 58d131ca819..24588405dbc 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -515,35 +515,20 @@ public async Task CompletionWrite_StorageOutcomeDecidesCommit(bool failWrite) } } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task CompletionWrite_HookOutcomePreservesActualCommit(bool postCommit) + [Fact] + public async Task CompletionWrite_PreparationFailureReplaysCompletion() { var rig = await CreateAsync(); using var handler = rig.Handler; var input = await DeliverAsync(rig); - var error = new IOException("Journal hook failed."); - await OnTurnAsync(rig.Context, () => - { - if (postCommit) rig.Grain.AfterJournalOperationFailure = error; - else rig.Grain.BeforeJournalOperationFailure = error; - }); + var error = new IOException("Journal preparation failed."); + await OnTurnAsync(rig.Context, () => rig.Grain.JournalPreparationFailure = error); var finished = FinishedAsync(rig); handler.Release.TrySetResult(); await WaitAsync(finished); - if (postCommit) - { - AssertSuccess(rig, input, outputCount: 0); - await OnTurnAsync(rig.Context, () => rig.Grain.AfterJournalOperationFailure = null); - await AssertHealthyAsync(rig); - } - else - { - var failure = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); - Assert.Same(error, failure.InnerException); - await AssertFailureReplayAsync(rig, input); - } + var failure = Assert.IsType(await WaitAsync(rig.Grain.DeactivationFailure.Task)); + Assert.Same(error, failure.InnerException); + await AssertFailureReplayAsync(rig, input); } [Fact] diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs index ba4ebecc285..d076f34d609 100644 --- a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -128,8 +128,6 @@ private sealed class TestSiloBuilder : ISiloBuilder private sealed class ConstructionTestStateManager : IJournaledStateManager { - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => default; - public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => default; public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) { } public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Messaging.Tests/Support/JournalProbeState.cs b/test/Orleans.Messaging.Tests/Support/JournalProbeState.cs new file mode 100644 index 00000000000..07252f4e688 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/JournalProbeState.cs @@ -0,0 +1,24 @@ +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +// An empty journal state which lets tests fail write preparation or observe write acknowledgement. +internal sealed class JournalProbeState : IStateMachine +{ + public const string Name = "test-journal-probe"; + + public JournalProbeState(IJournaledStateManager manager) => manager.RegisterStateMachine(Name, this); + + public Exception? PreparationFailure { get; set; } + public Action? WriteCompleted { get; set; } + + public ValueTask PrepareWriteAsync(CancellationToken cancellationToken = default) => + PreparationFailure is { } failure ? ValueTask.FromException(failure) : default; + + public void OnWriteCompleted() => WriteCompleted?.Invoke(); + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + throw new NotSupportedException("The journal probe has no entries."); + public void Reset(JournalStreamWriter writer) { } + public void WritePendingEntries(JournalStreamWriter writer) { } + public void WriteSnapshot(JournalStreamWriter writer) { } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 65c8bc46837..90c53f0f907 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -84,8 +84,12 @@ public sealed class MessagingTestGrain : DurableGrain, IMessagingTestGrain, IDur private readonly SerializerSessionPool _sessions; private readonly TestHandlerConfiguration _handlerConfiguration; internal IInboxHandler? HandlerOverride { get; set; } - internal Exception? BeforeJournalOperationFailure { get; set; } - internal Exception? AfterJournalOperationFailure { get; set; } + private readonly JournalProbeState _journalProbe; + internal Exception? JournalPreparationFailure + { + get => _journalProbe.PreparationFailure; + set => _journalProbe.PreparationFailure = value; + } private readonly ILocalSiloDetails _siloDetails; private readonly HandlerProbe _handlerProbe; private readonly SnapshotProbe _snapshotProbe; @@ -126,7 +130,7 @@ public MessagingTestGrain( _handlerProbe = handlerProbe; _snapshotProbe = snapshotProbe; _lifecycleSubscription = GrainLifecycleEvents.AllEvents.Subscribe(this); - _journalOwner.AddHookAsync(new ControlledJournalHook(this)).GetAwaiter().GetResult(); + _journalProbe = new JournalProbeState(journalOwner); } public override async Task OnActivateAsync(CancellationToken cancellationToken) @@ -443,21 +447,6 @@ public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken can context, cancellationToken); } } - - private sealed class ControlledJournalHook(MessagingTestGrain owner) : IJournaledStateHook - { - public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - if (owner.BeforeJournalOperationFailure is { } failure) throw failure; - return default; - } - - public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - if (owner.AfterJournalOperationFailure is { } failure) throw failure; - return default; - } - } } // Tests can change registration between real activations without adding a production registry API. From 0b8a193b82c66be9447a42ade22ac43defa76db9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 16:18:57 -0700 Subject: [PATCH 30/95] fix(messaging): start the pump loop asynchronously --- src/Orleans.Messaging/MessagingPump.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 8489a15ab64..841911ef635 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -131,6 +131,9 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte private async Task RunAsync() { + // Callers start the loop from grain calls and job executions; queue the first pass to the activation + // scheduler so it never runs on, and nests within, the caller's stack. + await Task.Yield(); var cancellationToken = _shutdown.Token; while (!cancellationToken.IsCancellationRequested) { From f3c9c56154c09bf7d5ad1f33eeabc7095727f678 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 18:09:52 -0700 Subject: [PATCH 31/95] test(messaging): remove handler races and simplify timeout reporting --- .../StateManagerPreparationTests.cs | 5 ++-- .../InboxHandlerTransactionTests.cs | 5 ++-- .../Functional/InboxLifecycleBehaviorTests.cs | 12 +++++--- .../MessagingRoutingAndDeadLetterTests.cs | 12 ++++---- .../Support/ControlledDurableJobManager.cs | 19 +++++++------ .../Support/SnapshotProbe.cs | 28 ++++++++++++++++++- 6 files changed, 56 insertions(+), 25 deletions(-) diff --git a/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs index fed449385bd..19ac4755ec3 100644 --- a/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerPreparationTests.cs @@ -113,7 +113,7 @@ await scheduler.Run(async () => [Theory] [InlineData(false)] [InlineData(true)] - public async Task Preparation_FailureRetainsPendingStateAndSnapshotForExplicitRetry(bool snapshot) + public async Task Preparation_FailureRetainsPendingStateForExplicitRetry(bool snapshot) { var storage = new CapturingStorage { IsCompactionRequested = snapshot }; await using var manager = CreateTestSystem(storage).Manager; @@ -133,7 +133,6 @@ public async Task Preparation_FailureRetainsPendingStateAndSnapshotForExplicitRe Assert.Equal(0, state.WriteCompletedCount); Assert.Empty(storage.Appends); Assert.Empty(storage.Replaces); - storage.IsCompactionRequested = false; state.Preparation = null; await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(31, state.CapturedValue); @@ -223,7 +222,7 @@ public async Task Preparation_ShutdownDrainsPreparationAndPreventsCapture() [Fact] public void PreCommitFailure_RoundTripsWithOriginalCause() { - var original = new JournaledStatePreCommitException(new IOException("Scheduling failed.")); + Exception original = new JournaledStatePreCommitException(new IOException("Scheduling failed.")); var serializer = ServiceProvider.GetRequiredService(); var copy = Assert.IsType(serializer.Deserialize(serializer.SerializeToArray(original))); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs index 4a0c937b01f..8da195a08bd 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerTransactionTests.cs @@ -116,15 +116,16 @@ public async Task HandlerCompletionWriteFailure_FencesOldManagerAndFreshActivati var oldContext = Fixture.GetGrainContext(receiver); var oldGrain = Assert.IsType(oldContext.GrainInstance); var oldManager = oldContext.ActivationServices.GetRequiredService(); + var oldActivationId = oldGrain.GetSnapshotForTest().ActivationId; Fixture.Storage.FailWrite(JournalId.FromGrainId(receiver.GetGrainId())); handler.Release(); await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); await Assert.ThrowsAnyAsync(() => oldManager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); _ = await receiver.GetSnapshotAsync(); var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), - snapshot => snapshot.ActivationId != oldGrain.GetSnapshotForTest().ActivationId + snapshot => snapshot.ActivationId != oldActivationId && snapshot.Effects.Sum(static effect => effect.Count) == 1); - Assert.NotEqual(oldGrain.GetSnapshotForTest().ActivationId, completed.ActivationId); + Assert.NotEqual(oldActivationId, completed.ActivationId); Assert.Equal(1, Assert.Single(completed.Effects).Count); Assert.Empty(completed.InboxDeadLetters); Assert.Equal(0, completed.InboxCount); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs index 31bb572d302..b4111bb5c55 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs @@ -43,10 +43,14 @@ public async Task DeactivationBeforeProcessing_RecoveryDrainsAcceptedMessageOnce var receiver = NewGrain(); var before = await receiver.GetSnapshotAsync(); var context = Fixture.GetGrainContext(receiver); - var envelope = CreateEnvelope(receiver, NewMessage(96, "queued")); - - Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); - await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var envelope = CreateEnvelope(receiver, NewMessage(96, "queued"), "messages/queued"); + + // Deactivation cancels any handler the original activation starts, leaving the message to recovery. + using (Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/queued")) + { + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync(envelope)).Status); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } _ = await receiver.GetSnapshotAsync(); var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs index ac917403d44..098ddca7be1 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs @@ -56,13 +56,13 @@ public async Task ActivationRemovesExpiredInboxDeadLetters() var before = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); Fixture.Clock.Advance(TimeSpan.FromHours(2)); - await receiver.RequestDeactivationAsync(); - var after = await Fixture.SnapshotProbe.WaitAsync( - receiver.GetGrainId(), - snapshot => snapshot.ActivationId != before.ActivationId - && snapshot.InboxDeadLetters.Count == 0); - + await ReactivateAsync(receiver); + var after = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, after.ActivationId); Assert.Empty(after.InboxDeadLetters); + + await ReactivateAsync(receiver); + Assert.Empty((await receiver.GetSnapshotAsync()).InboxDeadLetters); } [Fact] diff --git a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs index 77edf5e3dd8..9380f0b3c44 100644 --- a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs +++ b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs @@ -11,7 +11,7 @@ public sealed class DurableJobManagerProbe private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _successes = []; private readonly ConcurrentDictionary _failures = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _postScheduleFailures = new(StringComparer.Ordinal); - private readonly ConcurrentDictionary _duplicates = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _duplicates = []; private readonly ConcurrentDictionary<(string JobName, GrainId Target), ConcurrentQueue> _scheduledJobs = []; private readonly ConcurrentDictionary _scheduleBarriers = new(StringComparer.Ordinal); @@ -21,8 +21,8 @@ public void FailNext(string jobName) => public void FailAfterNext(string jobName) => _postScheduleFailures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); - public void DuplicateNext(string jobName) => - _duplicates.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + public void DuplicateNext(string jobName, GrainId target) => + _duplicates.AddOrUpdate((jobName, target), 1, static (_, count) => count + 1); public int GetAttemptCount(string jobName, GrainId target) => _attempts.TryGetValue((jobName, target), out var count) ? count : 0; @@ -59,8 +59,8 @@ internal bool ShouldFail(string jobName) internal bool ShouldFailAfterSchedule(string jobName) => TryConsumeFailure(_postScheduleFailures, jobName); - internal bool ShouldDuplicate(string jobName) - => TryConsumeFailure(_duplicates, jobName); + internal bool ShouldDuplicate(ScheduleJobRequest request) + => TryConsumeFailure(_duplicates, (request.JobName, request.Target)); internal async Task WaitIfBlockedAsync(string jobName, CancellationToken cancellationToken) { @@ -73,9 +73,10 @@ internal async Task WaitIfBlockedAsync(string jobName, CancellationToken cancell await barrier.Release.Task.WaitAsync(cancellationToken); } - private static bool TryConsumeFailure( - ConcurrentDictionary failures, - string jobName) + private static bool TryConsumeFailure( + ConcurrentDictionary failures, + TKey jobName) + where TKey : notnull { while (failures.TryGetValue(jobName, out var remaining) && remaining > 0) { @@ -118,7 +119,7 @@ public async Task ScheduleJobAsync( await probe.WaitIfBlockedAsync(request.JobName, cancellationToken); var result = await inner.ScheduleJobAsync(request, cancellationToken); probe.OnScheduled(request, result); - if (probe.ShouldDuplicate(request.JobName)) + if (probe.ShouldDuplicate(request)) { var duplicate = await inner.ScheduleJobAsync(request, cancellationToken); probe.OnScheduled(request, duplicate); diff --git a/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs index 81be9cbe4f3..7550f59ff90 100644 --- a/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs +++ b/test/Orleans.Messaging.Tests/Support/SnapshotProbe.cs @@ -6,6 +6,7 @@ namespace Orleans.Messaging.Tests.Support; public sealed class SnapshotProbe { private readonly ConcurrentDictionary _latest = new(); + private readonly ConcurrentDictionary> _history = new(); private readonly ConcurrentDictionary> _waiters = new(); internal int WaiterListCount => _waiters.Count; @@ -48,6 +49,11 @@ public async Task WaitAsync( { return await waiter.Completion.Task.WaitAsync(timeout ?? TimeSpan.FromSeconds(30)); } + catch (TimeoutException exception) + { + var published = _history.TryGetValue(grainId, out var history) ? string.Join(" | ", history) : "none"; + throw new TimeoutException($"Snapshot wait for '{grainId}' timed out. Published (oldest first): {published}", exception); + } finally { lock (waiters) @@ -61,6 +67,9 @@ public async Task WaitAsync( public void Publish(GrainId grainId, EndpointSnapshot snapshot) { _latest[grainId] = snapshot; + var history = _history.GetOrAdd(grainId, static _ => new()); + history.Enqueue(Describe(snapshot)); + while (history.Count > 8) history.TryDequeue(out _); if (!_waiters.TryGetValue(grainId, out var waiters)) { return; @@ -70,7 +79,19 @@ public void Publish(GrainId grainId, EndpointSnapshot snapshot) { foreach (var waiter in waiters.ToArray()) { - if (waiter.Predicate(snapshot)) + bool matched; + try + { + matched = waiter.Predicate(snapshot); + } + catch (Exception exception) + { + waiters.Remove(waiter); + waiter.Completion.TrySetException(exception); + continue; + } + + if (matched) { waiters.Remove(waiter); waiter.Completion.TrySetResult(snapshot); @@ -81,6 +102,11 @@ public void Publish(GrainId grainId, EndpointSnapshot snapshot) } } + internal static string Describe(EndpointSnapshot snapshot) => + $"activation={snapshot.ActivationId}, inbox={snapshot.InboxCount}, outbox={snapshot.OutboxCount}, " + + $"effects={snapshot.Effects.Sum(static effect => effect.Count)}, processed={snapshot.ProcessedMessageCount}, " + + $"inboxDeadLetters={snapshot.InboxDeadLetters.Count}, outboxDeadLetters={snapshot.OutboxDeadLetters.Count}"; + private void RemoveWaiterListIfEmpty(GrainId grainId, List waiters) { if (waiters.Count == 0) From c2e1f3ff5ffb1faac3e03d808710935d8c63f987 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 19:12:47 -0700 Subject: [PATCH 32/95] test(messaging): capture pump shutdown token before deactivation A failed completion write deactivates the grain and disposes its activation services, so resolving the pump inside the exception filter threw and let the cancellation escape. --- .../Functional/InboxHandlerCompletionTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs index 24588405dbc..c6aecb8d773 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -631,6 +631,8 @@ private async Task AssertFailureReplayAsync(Rig rig, InboxMessage input) /// private Task FinishedAsync(Rig rig) { + // Resolve the pump now: a failed write deactivates the grain and disposes its activation services. + var shutdown = rig.Context.ActivationServices.GetRequiredService().ShutdownToken; return RunAsync(); async Task RunAsync() @@ -639,7 +641,7 @@ async Task RunAsync() { await Fixture.RunPumpAsync(rig.Receiver); } - catch (OperationCanceledException) when (rig.Context.ActivationServices.GetRequiredService().ShutdownToken.IsCancellationRequested) + catch (OperationCanceledException) when (shutdown.IsCancellationRequested) { // A failed write stops the pump; callers assert the failure explicitly. } From 7190e6f11a4fe0603e31cf65bdb80073aee2965a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 20:09:30 -0700 Subject: [PATCH 33/95] test(messaging): hold the active handler through deactivation Deactivation_StopsPumpAndWaitsForActiveHandler armed a handler barrier that observed the shutdown token, so the handler exited on shutdown and deactivation could complete before the test asserted it was still pending. The barrier now ignores cancellation for this test, so deactivation waits until the test releases the handler. --- .../Functional/InboxLifecycleBehaviorTests.cs | 2 +- test/Orleans.Messaging.Tests/Support/HandlerProbe.cs | 12 +++++++++--- .../Support/MessagingTestGrains.cs | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs index b4111bb5c55..55870a5c602 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxLifecycleBehaviorTests.cs @@ -17,7 +17,7 @@ public async Task Deactivation_StopsPumpAndWaitsForActiveHandler() var before = await receiver.GetSnapshotAsync(); var context = Fixture.GetGrainContext(receiver); var shutdown = Fixture.GetPump(receiver).ShutdownToken; - using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/teardown-active"); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/teardown-active", observeCancellation: false); var envelope = CreateEnvelope(receiver, NewMessage(95, "active-handler"), "messages/teardown-active"); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); await handler.WaitUntilEnteredAsync(); diff --git a/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs index 0ecb2480c55..63e30f2c8da 100644 --- a/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs +++ b/test/Orleans.Messaging.Tests/Support/HandlerProbe.cs @@ -8,9 +8,9 @@ public sealed class HandlerProbe { private readonly ConcurrentDictionary<(GrainId GrainId, string Route), Barrier> _barriers = new(); - public Barrier Arm(GrainId grainId, string route) + public Barrier Arm(GrainId grainId, string route, bool observeCancellation = true) { - var barrier = new Barrier(this, (grainId, route)); + var barrier = new Barrier(this, (grainId, route), observeCancellation); if (!_barriers.TryAdd((grainId, route), barrier)) { throw new InvalidOperationException($"A handler barrier is already armed for '{grainId}' and route '{route}'."); @@ -29,11 +29,13 @@ public sealed class Barrier : IDisposable { private readonly HandlerProbe _owner; private readonly (GrainId GrainId, string Route) _key; + private readonly bool _observeCancellation; - internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key) + internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key, bool observeCancellation) { _owner = owner; _key = key; + _observeCancellation = observeCancellation; } internal TaskCompletionSource Entered { get; } = @@ -44,6 +46,10 @@ internal Barrier(HandlerProbe owner, (GrainId GrainId, string Route) key) public Task WaitUntilEnteredAsync() => Entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); public void Release() => Continue.TrySetResult(); + // A barrier that ignores cancellation holds the handler until Release, even after shutdown. + internal Task WaitAsync(CancellationToken cancellationToken) => + _observeCancellation ? Continue.Task.WaitAsync(cancellationToken) : Continue.Task; + public void Dispose() { Release(); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 90c53f0f907..a734f03e191 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -357,7 +357,7 @@ private async ValueTask HandleAsync( if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route, out var gate)) { gate.Entered.TrySetResult(); - await gate.Continue.Task.WaitAsync(cancellationToken); + await gate.WaitAsync(cancellationToken); } if (_handlerProbe.TryGet(this.GetGrainId(), TestApplicationProtocol.Read(_sessions, context.Message).Route + "/application-preparation", out var preparation)) From 488522dc24d0f5f65ed1535099092899dfe5ec00 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 20:58:44 -0700 Subject: [PATCH 34/95] fix(messaging): address inbox review feedback Fail the activation when a pump pass or wake-up repair fails unexpectedly, let DeliverAsync callers stop waiting on cancellation while the owned write continues, validate recovered dead-letter keys, skip grain types without an implementation class, and make DeadLetter serializable for diagnostics returned over RPC. --- .../IMessagingDiagnostics.cs | 5 +++ src/Orleans.Messaging/Inbox.State.cs | 10 +++++- src/Orleans.Messaging/Inbox.cs | 21 ++++++++++-- .../MessagingGrainTypeConfigurator.cs | 8 ++--- src/Orleans.Messaging/MessagingPump.cs | 17 +++++++++- .../Orleans.Messaging/Orleans.Messaging.cs | 34 +++++++++++++++++++ .../Contracts/EnvelopeContractTests.cs | 23 +++++++++++++ .../Functional/InboxStateEncodingTests.cs | 27 +++++++++++++++ 8 files changed, 134 insertions(+), 11 deletions(-) diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs index ea28b65e64d..b2c1e245e26 100644 --- a/src/Orleans.Messaging/IMessagingDiagnostics.cs +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -30,6 +30,7 @@ public interface IMessagingDiagnostics /// Describes a dead-lettered durable message. /// /// The incoming or outgoing message type. +[GenerateSerializer, Alias("Orleans.Messaging.DeadLetter`1")] public sealed class DeadLetter { /// @@ -39,21 +40,25 @@ public sealed class DeadLetter /// Treat the message's encoded headers as immutable. Application diagnostic DTOs use /// ordinary RPC copying when returning message buffers. /// + [Id(0)] public required TMessage Message { get; init; } /// /// Gets when the message was dead-lettered. /// + [Id(1)] public DateTimeOffset DeadLetteredAt { get; init; } /// /// Gets the terminal failure reason. /// + [Id(2)] public required string Reason { get; init; } /// /// Gets the number of attempts made. /// + [Id(3)] public int AttemptCount { get; init; } } diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index e496b3adc0f..79cd8bf4260 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -63,7 +63,15 @@ void IStateMachine.OnRecoveryCompleted() } } - foreach (var letter in _deadLetters.Values) EnvelopeValidation.Validate(letter.Message); + foreach (var (key, letter) in _deadLetters) + { + EnvelopeValidation.Validate(letter.Message); + if (key != letter.Message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered inbox dead-letter key does not match its command identity."); + } + } + _capturedSequence = _durableSequence = _lastSequence; } diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index f0eaef7c39a..32daf30f254 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -79,7 +79,7 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel } // The sender may retire its copy once this returns, so the acceptance must be durable. - if (!IsDurable(pending)) await PersistAsync().ConfigureAwait(true); + if (!IsDurable(pending)) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); _pump.Signal(); Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); return DeliveryResult.Duplicate(); @@ -99,12 +99,12 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel } Emit(new() { Kind = InboxCommandKind.Accept, Message = message }); - await PersistAsync().ConfigureAwait(true); + await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); LogMessageAccepted(Logger, key, GrainContext.GrainId); return DeliveryResult.Accepted(); } - catch (Exception exception) + catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested) { LogDeliveryOperationFailed(Logger, exception, key, GrainContext.GrainId); throw; @@ -318,6 +318,21 @@ internal async Task StartAsync(CancellationToken cancellationToken) private void ReportDepth(int delta) => Shared.Instruments.OnInboxDepthChanged(Shared.GrainType, delta); + /// Persists staged changes while ends only the caller's wait. + private ValueTask PersistForCallerAsync(CancellationToken cancellationToken) + { + var write = PersistAsync(); + if (write.IsCompleted || !cancellationToken.CanBeCanceled) + { + return write; + } + + // The write continues to its outcome after the caller stops waiting; PersistAsync reports failures through the pump. + var task = write.AsTask(); + task.Ignore(); + return new(task.WaitAsync(cancellationToken)); + } + private async ValueTask PersistAsync() { try diff --git a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs index 81d6460fcdf..b34ae02031c 100644 --- a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs +++ b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs @@ -10,12 +10,8 @@ internal sealed class MessagingGrainTypeConfigurator(GrainClassMap grainClasses, { public void Configure(GrainType grainType, GrainProperties properties, GrainTypeSharedContext shared) { - if (!grainClasses.TryGetGrainClass(grainType, out var grainClass)) - { - throw new InvalidOperationException($"No grain implementation is registered for '{grainType}'."); - } - - if (typeof(DurableGrain).IsAssignableFrom(grainClass) || typeof(IMessagingGrain).IsAssignableFrom(grainClass)) + if (grainClasses.TryGetGrainClass(grainType, out var grainClass) + && (typeof(DurableGrain).IsAssignableFrom(grainClass) || typeof(IMessagingGrain).IsAssignableFrom(grainClass))) { var placementStrategy = shared.PlacementStrategy; shared.SetComponent(new MessagingShared(services, grainType)); diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 841911ef635..521be7c7484 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -113,7 +113,7 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte _wakeupScheduled = false; if (!_shutdown.IsCancellationRequested && Inbox.Count > 0) { - EnsureWakeupAsync(CancellationToken.None).AsTask().Ignore(); + RepairWakeupAsync().Ignore(); } throw; @@ -129,6 +129,19 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte MessagingTime.AddClamped(Shared.JobTimeProvider.GetUtcNow(), delay > TimeSpan.Zero ? delay : RecheckDelay)); } + private async Task RepairWakeupAsync() + { + try + { + await EnsureWakeupAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (Exception exception) when (!_shutdown.IsCancellationRequested) + { + // Recovery re-establishes the wake-up for remaining work. + Fail(exception, "Scheduling the messaging wake-up failed."); + } + } + private async Task RunAsync() { // Callers start the loop from grain calls and job executions; queue the first pass to the activation @@ -149,8 +162,10 @@ private async Task RunAsync() } catch (Exception exception) { + // Expected failures are handled within the pass, so an escaping exception leaves state inconsistent with the journal. progress = false; LogPassFailed(Shared.PumpLogger, exception, GrainContext.GrainId); + Fail(exception, "Messaging processing pass failed."); } pass?.TrySetResult(); diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 37a59fbf816..2b5287cae4e 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -23,14 +23,20 @@ public static partial class MessagingExtensions namespace Orleans.Messaging { + [GenerateSerializer] + [Alias("Orleans.Messaging.DeadLetter`1")] public sealed partial class DeadLetter { + [Id(3)] public int AttemptCount { get { throw null; } init { } } + [Id(1)] public System.DateTimeOffset DeadLetteredAt { get { throw null; } init { } } + [Id(0)] public required TMessage Message { get { throw null; } init { } } + [Id(2)] public required string Reason { get { throw null; } init { } } } @@ -367,6 +373,24 @@ public void Validate() { } namespace OrleansCodeGen.Orleans.Messaging { + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_DeadLetter : global::Orleans.Serialization.Codecs.IFieldCodec>, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_DeadLetter(global::Orleans.Serialization.Activators.IActivator> _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Messaging.DeadLetter instance) { } + + public global::Orleans.Messaging.DeadLetter ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Messaging.DeadLetter instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Messaging.DeadLetter value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] @@ -457,6 +481,16 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. where TBufferWriter : System.Buffers.IBufferWriter { } } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_DeadLetter : global::Orleans.Serialization.Cloning.IDeepCopier>, global::Orleans.Serialization.Cloning.IDeepCopier + { + public Copier_DeadLetter(global::Orleans.Serialization.Activators.IActivator> _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public global::Orleans.Messaging.DeadLetter DeepCopy(global::Orleans.Messaging.DeadLetter original, global::Orleans.Serialization.Cloning.CopyContext context) { throw null; } + } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 237b69bd9ee..6c76e3582a6 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -202,6 +202,29 @@ public void MessageAndDeadLetterCopyPreservesMetadataAndIsolation(string kind) } } + [Fact] + public void PublicDeadLetterRoundTripsThroughOrleansSerialization() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var letter = new DeadLetter + { + Message = new InboxMessage { Envelope = Create([1, 2]) with { Subject = "subject.v1" } }, + DeadLetteredAt = DateTimeOffset.UnixEpoch, + Reason = "failure", + AttemptCount = 3 + }; + var serializer = services.GetRequiredService>>(); + + var result = Assert.IsType>(serializer.Deserialize(serializer.SerializeToArray(letter))); + + Assert.Equal(letter.Message.Envelope.MessageId, result.Message.Envelope.MessageId); + Assert.Equal(new byte[] { 1, 2 }, result.Message.Envelope.Payload); + Assert.Equal("subject.v1", result.Message.Envelope.Subject); + Assert.Equal(DateTimeOffset.UnixEpoch, result.DeadLetteredAt); + Assert.Equal("failure", result.Reason); + Assert.Equal(3, result.AttemptCount); + } + [Fact] public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() { diff --git a/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs index fc8f6a2d9ed..7e06d07ee33 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxStateEncodingTests.cs @@ -47,6 +47,33 @@ public async Task CompleteCommandCodecFailure_PreservesOriginalCauseAndReplaysAc Assert.Empty(completed.InboxDeadLetters); } + [Fact] + public async Task HandlerErrorOutcomeCodecFailure_DeactivatesAndReplaysAcceptedMessage() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/error-codec-fault"); + var message = NewMessage(205, "error-codec-fault") with { ThrowOnceDuringPreparation = true }; + var envelope = CreateEnvelope(receiver, message, "messages/error-codec-fault"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var failure = new IOException("Handler error outcome command failed."); + ((FaultingInboxCodecFixture)Fixture).NextFailure = failure; + handler.Release(); + Assert.Same(failure, await grain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Empty(completed.InboxDeadLetters); + } + [Theory] [InlineData(false)] [InlineData(true)] From 95b0d98bc826586fe22fb7273268452392d0a59c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 21:19:59 -0700 Subject: [PATCH 35/95] fix(messaging): discard wake-ups overtaken by a completed job run A pump job can run and complete before its scheduling call returns. Track a wake-up epoch which advances whenever a job run clears the scheduled wake-up, and record a schedule as a guaranteed wake-up only when no run completed while it was in flight. --- src/Orleans.Messaging/MessagingPump.cs | 34 ++++++++++++++----- .../Functional/InboxRetryTests.cs | 25 ++++++++++++++ .../Support/ControlledDurableJobManager.cs | 20 +++++++++++ 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 521be7c7484..17ee681fd7c 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -28,6 +28,11 @@ internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecy private Task? _scheduling; private bool _wakeupScheduled; + // Advances whenever a job run stops guaranteeing a future wake-up. A schedule requested in an earlier epoch may + // belong to a job which has already run and completed, so it no longer guarantees a wake-up. + private int _wakeupEpoch; + private int _schedulingEpoch; + public MessagingPump(IGrainContext grainContext, IJournaledStateManager stateManager, IDurableJobHandlerRegistry jobHandlers) { GrainContext = grainContext; @@ -79,16 +84,23 @@ public void EnsureStarted() public async ValueTask EnsureWakeupAsync(CancellationToken cancellationToken) { if (_wakeupScheduled) return; - var scheduling = _scheduling ??= Shared.JobManager.ScheduleJobAsync(new ScheduleJobRequest + if (_scheduling is null || _schedulingEpoch != _wakeupEpoch) { - Target = GrainContext.GrainId, - JobName = JobName, - DueTime = Shared.JobTimeProvider.GetUtcNow() - }, _shutdown.Token); + _schedulingEpoch = _wakeupEpoch; + _scheduling = Shared.JobManager.ScheduleJobAsync(new ScheduleJobRequest + { + Target = GrainContext.GrainId, + JobName = JobName, + DueTime = Shared.JobTimeProvider.GetUtcNow() + }, _shutdown.Token); + } + + var scheduling = _scheduling; + var epoch = _schedulingEpoch; try { await scheduling.WaitAsync(cancellationToken).ConfigureAwait(true); - _wakeupScheduled = true; + if (epoch == _wakeupEpoch) _wakeupScheduled = true; } finally { @@ -110,7 +122,7 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte catch { // This attempt no longer guarantees a future wake-up. - _wakeupScheduled = false; + ClearWakeup(); if (!_shutdown.IsCancellationRequested && Inbox.Count > 0) { RepairWakeupAsync().Ignore(); @@ -121,7 +133,7 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte if (Inbox.GetNextWakeupDelay() is not { } delay) { - _wakeupScheduled = false; + ClearWakeup(); return DurableJobRunResult.Completed; } @@ -129,6 +141,12 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte MessagingTime.AddClamped(Shared.JobTimeProvider.GetUtcNow(), delay > TimeSpan.Zero ? delay : RecheckDelay)); } + private void ClearWakeup() + { + _wakeupScheduled = false; + _wakeupEpoch++; + } + private async Task RepairWakeupAsync() { try diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs index 98d1ec2bc91..5d3616d37f0 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -122,6 +122,31 @@ public async Task AcceptedMessage_WaitsForHandlerRegistrationWithoutDeadLetterin Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); } + [Fact] + public async Task WakeupWhoseJobCompletesBeforeSchedulingReturns_DoesNotSuppressLaterWakeups() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var pump = Fixture.GetPump(receiver); + var target = receiver.GetGrainId(); + + // The scheduled job runs and completes, with no work remaining, before the scheduling call returns. + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + using var schedule = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); + var scheduling = await Fixture.OnGrainAsync(receiver, () => pump.EnsureWakeupAsync(TestContext.Current.CancellationToken).AsTask()); + await schedule.WaitUntilEnteredAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + schedule.Continue(); + await scheduling.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + + // Later work schedules its own wake-up. + var attempts = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, target); + var envelope = CreateEnvelope(receiver, NewMessage(195, "later-work")); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(attempts + 1, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, target)); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + } + private sealed class RetryFixture : MessagingClusterFixture { protected override void ConfigureOptions(DurableMessagingOptions options) diff --git a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs index 9380f0b3c44..d1831ad66eb 100644 --- a/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs +++ b/test/Orleans.Messaging.Tests/Support/ControlledDurableJobManager.cs @@ -12,6 +12,7 @@ public sealed class DurableJobManagerProbe private readonly ConcurrentDictionary _failures = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _postScheduleFailures = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<(string JobName, GrainId Target), int> _duplicates = []; + private readonly ConcurrentDictionary _skips = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<(string JobName, GrainId Target), ConcurrentQueue> _scheduledJobs = []; private readonly ConcurrentDictionary _scheduleBarriers = new(StringComparer.Ordinal); @@ -21,6 +22,10 @@ public void FailNext(string jobName) => public void FailAfterNext(string jobName) => _postScheduleFailures.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + /// Returns the next schedule without storing its job, modeling a job which has already run. + public void SkipNext(string jobName) => + _skips.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + public void DuplicateNext(string jobName, GrainId target) => _duplicates.AddOrUpdate((jobName, target), 1, static (_, count) => count + 1); @@ -59,6 +64,9 @@ internal bool ShouldFail(string jobName) internal bool ShouldFailAfterSchedule(string jobName) => TryConsumeFailure(_postScheduleFailures, jobName); + internal bool ShouldSkip(string jobName) + => TryConsumeFailure(_skips, jobName); + internal bool ShouldDuplicate(ScheduleJobRequest request) => TryConsumeFailure(_duplicates, (request.JobName, request.Target)); @@ -117,6 +125,18 @@ public async Task ScheduleJobAsync( } await probe.WaitIfBlockedAsync(request.JobName, cancellationToken); + if (probe.ShouldSkip(request.JobName)) + { + return new DurableJob + { + Id = Guid.NewGuid().ToString("N"), + Name = request.JobName, + ShardId = "skipped", + TargetGrainId = request.Target, + DueTime = request.DueTime + }; + } + var result = await inner.ScheduleJobAsync(request, cancellationToken); probe.OnScheduled(request, result); if (probe.ShouldDuplicate(request)) From 2dcb7a99dca0da592fd71a919a802c8ef2b5d9a2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 21:48:52 -0700 Subject: [PATCH 36/95] feat(messaging): process inbox messages at DeliverAt The inbox accepts, deduplicates, and journals a message with Envelope.DeliverAt, then processes it once it is durable and the Durable Jobs clock reaches DeliverAt. The pump reschedules its durable wake-up for the earliest due message, and scheduled messages never count toward MaxCapacity. --- .../Configuration/DurableMessagingOptions.cs | 4 + src/Orleans.Messaging/Inbox.State.cs | 20 +- src/Orleans.Messaging/Inbox.cs | 30 ++- src/Orleans.Messaging/MessagingPump.cs | 5 +- .../MessagingCommandSerializationTests.cs | 34 +++- .../Functional/ScheduledDeliveryTests.cs | 188 ++++++++++++++++++ .../Support/ManualJobClock.cs | 17 ++ 7 files changed, 282 insertions(+), 16 deletions(-) create mode 100644 test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/ManualJobClock.cs diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index 35dbe4ec2c6..57aedc62e39 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -43,6 +43,10 @@ public class DurableMessagingOptions /// The inbox capacity is checked before accepting new messages. Messages are persisted to durable /// storage, so capacity limits affect both in-memory state and storage I/O during recovery. /// + /// + /// Messages with an time do not count toward this limit, including after they + /// become due, and are accepted regardless of it. + /// /// /// /// The maximum inbox capacity. Must be greater than zero. Defaults to 1000. diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index 79cd8bf4260..346e1ea7340 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -15,6 +15,9 @@ internal sealed partial class Inbox private readonly Dictionary _deadLetters = []; private JournalStreamWriter _writer; + // Pending messages with a delivery time, which do not count toward MaxCapacity. + private int _scheduledCount; + // Entries are durable once a completed write captured their sequence number. private long _lastSequence; private long _capturedSequence; @@ -46,6 +49,7 @@ void IStateMachine.Reset(JournalStreamWriter writer) _messages.Clear(); _processed.Clear(); _deadLetters.Clear(); + _scheduledCount = 0; _lastSequence = _capturedSequence = _durableSequence = 0; } @@ -109,13 +113,19 @@ private void Apply(InboxCommand command) var message = command.Message ?? throw new InvalidOperationException("An accept command requires its message."); _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = ++_lastSequence }); _processed.Remove(message.Envelope.MessageId); + if (message.Envelope.DeliverAt is not null) _scheduledCount++; ReportDepth(1); break; case InboxCommandKind.Retry: _messages[command.MessageId].State = command.State ?? throw new InvalidOperationException("A retry command requires its state."); break; case InboxCommandKind.Complete: - if (_messages.Remove(command.MessageId)) ReportDepth(-1); + if (_messages.Remove(command.MessageId, out var completed)) + { + if (completed.Message.Envelope.DeliverAt is not null) _scheduledCount--; + ReportDepth(-1); + } + _processed[command.MessageId] = command.ProcessedAt; if (command.DeadLetter is { } letter) _deadLetters[command.MessageId] = letter; break; @@ -128,7 +138,13 @@ private void Apply(InboxCommand command) _messages.Clear(); _processed.Clear(); _deadLetters.Clear(); - foreach (var restored in snapshot.Messages) _messages.Add(restored.Message.Envelope.MessageId, restored); + _scheduledCount = 0; + foreach (var restored in snapshot.Messages) + { + _messages.Add(restored.Message.Envelope.MessageId, restored); + if (restored.Message.Envelope.DeliverAt is not null) _scheduledCount++; + } + ReportDepth(_messages.Count - previousCount); foreach (var processed in snapshot.Processed) _processed.Add(processed.MessageId, processed.ProcessedAt); diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 32daf30f254..f8c0fb43d1f 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -18,7 +18,8 @@ namespace Orleans.Messaging; /// /// /// All state transitions run on the activation scheduler, so each check-then-emit sequence is atomic. -/// Accepted messages are processed only after their acceptance is durable. +/// Accepted messages are processed only after their acceptance is durable and their , +/// if any, has passed. /// internal sealed partial class Inbox : IInbox, IInboxExtension, IStateMachine, IDurableValueCommandHandler { @@ -85,7 +86,8 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel return DeliveryResult.Duplicate(); } - if (_messages.Count >= Options.MaxCapacity) + // Scheduled messages never count toward capacity, so they are not backpressured either. + if (message.Envelope.DeliverAt is null && _messages.Count - _scheduledCount >= Options.MaxCapacity) { Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "backpressured"); return DeliveryResult.Backpressured(); @@ -122,10 +124,11 @@ internal async ValueTask ProcessReadyAsync(CancellationToken cancellationT } var now = TimeProvider.GetUtcNow(); + var jobNow = Shared.JobTimeProvider.GetUtcNow(); var batch = new List(Math.Min(Options.InboxBatchSize, _messages.Count)); foreach (var entry in _messages.Values) { - if (IsDurable(entry) && (entry.State.NextAttemptAt is not { } at || at <= now)) + if (IsDurable(entry) && GetDelay(entry, now, jobNow) == TimeSpan.Zero) { batch.Add(entry); if (batch.Count == Options.InboxBatchSize) break; @@ -285,18 +288,29 @@ internal bool RemoveDeadLetter(HierarchicalKey messageId) /// /// Returns the delay until the next message is due, or when the inbox is empty. /// - internal TimeSpan? GetNextWakeupDelay() + /// The current Durable Jobs time, which the caller uses to schedule the wake-up. + internal TimeSpan? GetNextWakeupDelay(DateTimeOffset jobNow) { if (_messages.Count == 0) return null; var now = TimeProvider.GetUtcNow(); - var next = DateTimeOffset.MaxValue; + var next = TimeSpan.MaxValue; foreach (var entry in _messages.Values) { - if (entry.State.NextAttemptAt is not { } at || at <= now) return TimeSpan.Zero; - if (at < next) next = at; + var delay = GetDelay(entry, now, jobNow); + if (delay == TimeSpan.Zero) return delay; + if (delay < next) next = delay; } - return next - now; + return next; + } + + // DeliverAt is in the Durable Jobs time domain, which schedules wake-ups; retry backoff follows the grain clock. + private static TimeSpan GetDelay(InboxEntry entry, DateTimeOffset now, DateTimeOffset jobNow) + { + var delay = TimeSpan.Zero; + if (entry.Message.Envelope.DeliverAt is { } deliverAt && deliverAt > jobNow) delay = deliverAt - jobNow; + if (entry.State.NextAttemptAt is { } retryAt && retryAt > now && retryAt - now > delay) delay = retryAt - now; + return delay; } internal async Task StartAsync(CancellationToken cancellationToken) diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 17ee681fd7c..fa88c4be84e 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -131,14 +131,15 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte throw; } - if (Inbox.GetNextWakeupDelay() is not { } delay) + var now = Shared.JobTimeProvider.GetUtcNow(); + if (Inbox.GetNextWakeupDelay(now) is not { } delay) { ClearWakeup(); return DurableJobRunResult.Completed; } return DurableJobRunResult.RescheduleAt( - MessagingTime.AddClamped(Shared.JobTimeProvider.GetUtcNow(), delay > TimeSpan.Zero ? delay : RecheckDelay)); + MessagingTime.AddClamped(now, delay > TimeSpan.Zero ? delay : RecheckDelay)); } private void ClearWakeup() diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 0e15996db72..6079c06a552 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -31,14 +31,39 @@ public void InboxCommand_RoundTripsThroughJsonAndOrleansSerialization(int index) Assert.Equal(json, JsonSerializer.Serialize(fromBinary!, InboxJsonContext.Default.InboxCommand)); } - private static InboxMessage InboxMessage(string id) => new() { Envelope = Envelope(id) }; + [Fact] + public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWhenUnset() + { + var deliverAt = new DateTimeOffset(2040, 1, 2, 3, 4, 5, TimeSpan.FromHours(2)); + var scheduled = new InboxCommand { Kind = InboxCommandKind.Accept, Message = InboxMessage("scheduled", deliverAt) }; + var immediate = new InboxCommand { Kind = InboxCommandKind.Accept, Message = InboxMessage("immediate") }; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + + var json = JsonSerializer.Serialize(scheduled, InboxJsonContext.Default.InboxCommand); + var fromJson = JsonSerializer.Deserialize(json, InboxJsonContext.Default.InboxCommand)!.Message!.Value.Envelope.DeliverAt; + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(scheduled))!.Message!.Value.Envelope.DeliverAt; + var immediateJson = JsonSerializer.Serialize(immediate, InboxJsonContext.Default.InboxCommand); + + Assert.Contains("\"DeliverAt\":", json, StringComparison.Ordinal); + Assert.Equal(deliverAt, fromJson); + Assert.Equal(deliverAt.Offset, fromJson!.Value.Offset); + Assert.Equal(deliverAt, fromBinary); + Assert.Equal(deliverAt.Offset, fromBinary!.Value.Offset); + Assert.DoesNotContain("DeliverAt", immediateJson, StringComparison.Ordinal); + Assert.Null(JsonSerializer.Deserialize(immediateJson, InboxJsonContext.Default.InboxCommand)!.Message!.Value.Envelope.DeliverAt); + Assert.Null(serializer.Deserialize(serializer.SerializeToArray(immediate))!.Message!.Value.Envelope.DeliverAt); + } + + private static InboxMessage InboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt) }; - private static Envelope Envelope(string id) => new() + private static Envelope Envelope(string id, DateTimeOffset? deliverAt = null) => new() { MessageId = HierarchicalKey.Create("orders", id), Payload = [1, 2, 3], Subject = "orders.v1", - Headers = new(StringComparer.Ordinal) { ["trace"] = [4, 5] } + Headers = new(StringComparer.Ordinal) { ["trace"] = [4, 5] }, + DeliverAt = deliverAt }; private static InboxCommand[] CreateInboxCommands() @@ -47,6 +72,7 @@ private static InboxCommand[] CreateInboxCommands() return [ new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("accept"), State = new() }, + new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("scheduled", Now), State = new() }, new() { Kind = InboxCommandKind.Retry, MessageId = HierarchicalKey.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom" } }, new() { Kind = InboxCommandKind.Complete, MessageId = HierarchicalKey.Create("orders", "done"), ProcessedAt = Now, DeadLetter = deadLetter }, new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [HierarchicalKey.Create("orders", "dead")] }, @@ -55,7 +81,7 @@ private static InboxCommand[] CreateInboxCommands() Kind = InboxCommandKind.Snapshot, Snapshot = new() { - Messages = [new() { Message = InboxMessage("pending"), State = new() { AttemptCount = 1 } }], + Messages = [new() { Message = InboxMessage("pending"), State = new() { AttemptCount = 1 } }, new() { Message = InboxMessage("scheduled", Now), State = new() }], Processed = [new() { MessageId = HierarchicalKey.Create("orders", "done"), ProcessedAt = Now }], DeadLetters = [deadLetter] } diff --git a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs new file mode 100644 index 00000000000..e0e30fb2676 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs @@ -0,0 +1,188 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class ScheduledDeliveryTests() : MessagingBehaviorTestBase(new ScheduledDeliveryClusterFixture()) +{ + private const string HeldRoute = "messages/held"; + private static readonly TimeSpan Delay = TimeSpan.FromHours(1); + + private ManualJobClock JobClock => ((ScheduledDeliveryClusterFixture)Fixture).JobClock; + + [Fact] + public async Task ScheduledMessage_IsHeldUntilDeliverAtThenProcessedOnce() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(1, "scheduled")), deliverAt); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + + // The acceptance is durable, but wake-ups before the due time leave the message pending and reschedule for it. + await AssertWakeupAtAsync(receiver, deliverAt); + await AssertWakeupAtAsync(receiver, deliverAt); + var pending = await receiver.GetSnapshotAsync(); + Assert.Equal(1, pending.InboxCount); + Assert.Empty(pending.Effects); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + JobClock.Advance(Delay - TimeSpan.FromTicks(1)); + await AssertWakeupAtAsync(receiver, deliverAt); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + + JobClock.Advance(TimeSpan.FromTicks(1)); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("scheduled", Assert.Single(completed.Effects).Value); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + + // Late and duplicate wake-ups and redeliveries after completion change nothing. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ScheduledMessage_SurvivesReactivationAndRestoresCapacityAccounting(bool snapshot) + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + if (snapshot) + { + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + } + + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(2, "recovered")), deliverAt); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + + await ReactivateAsync(receiver); + + var recovered = await receiver.GetSnapshotAsync(); + Assert.Equal(1, recovered.InboxCount); + Assert.Empty(recovered.Effects); + var inbox = Fixture.GetInbox(receiver); + var recoveredDeliverAt = await Fixture.OnGrainAsync(receiver, + () => InboxTestAccess.GetPendingMessages(inbox)[envelope.Envelope.MessageId].Message.Envelope.DeliverAt); + Assert.Equal(deliverAt, recoveredDeliverAt); + await AssertWakeupAtAsync(receiver, deliverAt); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + + // The recovered scheduled message does not count toward MaxCapacity (2). + using (var held = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute)) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(3, "held"), HeldRoute))).Status); + await WaitForBarrierAsync(receiver, held); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(4, "second")))).Status); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(5, "third")))).Status); + } + + var unscheduled = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.DoesNotContain(unscheduled.Effects, effect => effect.Value == "recovered"); + + JobClock.Advance(Delay); + _ = await Fixture.RunPumpAsync(receiver); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 3); + Assert.Equal(new[] { "recovered", "held", "second" }, completed.Effects.Select(static effect => effect.Value)); + Assert.All(completed.Effects, static effect => Assert.Equal(1, effect.Count)); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task ScheduledMessages_DoNotCountTowardCapacityEvenWhenDue() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + + // MaxCapacity is 2; scheduled messages are admitted beyond it. + for (var sequence = 10; sequence < 13; sequence++) + { + var scheduled = Schedule(CreateEnvelope(receiver, NewMessage(sequence, $"scheduled-{sequence}")), deliverAt); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, scheduled)).Status); + } + + var rejected = CreateEnvelope(receiver, NewMessage(22, "backpressured")); + using (var held = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute)) + { + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(20, "held"), HeldRoute))).Status); + await WaitForBarrierAsync(receiver, held); + + // The held pass has not processed the scheduled messages, which are now due and still pending. + JobClock.Advance(Delay); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, NewMessage(21, "second")))).Status); + Assert.Equal(5, (await receiver.GetSnapshotAsync()).InboxCount); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(receiver, rejected)).Status); + + // A due scheduled message is still accepted while unscheduled messages fill the inbox. + var due = Schedule(CreateEnvelope(receiver, NewMessage(13, "scheduled-13")), JobClock.GetUtcNow()); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, due)).Status); + Assert.Equal(6, (await receiver.GetSnapshotAsync()).InboxCount); + } + + var completed = await Fixture.WaitForEffectCountAsync(receiver, 6); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, rejected)).Status); + var all = await Fixture.WaitForEffectCountAsync(receiver, 7); + Assert.All(all.Effects, static effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task ScheduledMessage_RedeliveryCoalescesAndDifferentDeliverAtConflicts() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(30, "identity")), deliverAt); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(DeliveryStatus.Duplicate, + (await DeliverAsync(receiver, Schedule(envelope, deliverAt.ToOffset(TimeSpan.FromHours(-7))))).Status); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, Schedule(envelope, deliverAt + TimeSpan.FromSeconds(1)))); + await Assert.ThrowsAsync(() => DeliverAsync(receiver, Schedule(envelope, null))); + Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + var inbox = Fixture.GetInbox(receiver); + Assert.Equal(deliverAt, await Fixture.OnGrainAsync(receiver, + () => InboxTestAccess.GetPendingMessages(inbox)[envelope.Envelope.MessageId].Message.Envelope.DeliverAt)); + + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); + + // Completed IDs are deduplicated regardless of content. + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, Schedule(envelope, deliverAt + Delay))).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + + private async Task AssertWakeupAtAsync(IMessagingTestGrain grain, DateTimeOffset deliverAt) + { + var result = await Fixture.RunPumpAsync(grain); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, result.Status); + Assert.Equal(deliverAt, result.RescheduleTime); + } + + private static InboxMessage Schedule(InboxMessage message, DateTimeOffset? deliverAt) => + new() { Envelope = message.Envelope with { DeliverAt = deliverAt } }; + + private sealed class ScheduledDeliveryClusterFixture() : MessagingClusterFixture(1, receiverOnly: true) + { + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); + + protected override void ConfigureServices(IServiceCollection services) => + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs b/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs new file mode 100644 index 00000000000..a3331d2fdc0 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs @@ -0,0 +1,17 @@ +namespace Orleans.Messaging.Tests.Support; + +/// +/// A Durable Jobs clock whose current time moves only when a test advances it. +/// +/// +/// Timers and timestamps keep following real time, so Durable Jobs keeps running wake-ups that are due, +/// while tests decide exactly when scheduled messages become due. +/// +public sealed class ManualJobClock(DateTimeOffset start) : TimeProvider +{ + private long _utcTicks = start.UtcTicks; + + public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _utcTicks), TimeSpan.Zero); + + public void Advance(TimeSpan delta) => Interlocked.Add(ref _utcTicks, delta.Ticks); +} From 1b115ca2bf152c8de577f4340bae786bc4cdf74e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 22:37:30 -0700 Subject: [PATCH 37/95] test(messaging): fail once per message across reactivations --- test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index a734f03e191..1446ae3b8ab 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -96,7 +96,8 @@ internal Exception? JournalPreparationFailure private readonly Guid _activationId = Guid.NewGuid(); private int _activeHandlers; private int _maxConcurrentHandlers; - private readonly HashSet _failedOnce = []; + // Spans activations so a replayed message does not fail again after recovery. + private static readonly ConcurrentDictionary FailedOnce = new(); public MessagingTestGrain( IGrainContext grainContext, @@ -370,7 +371,7 @@ private async ValueTask HandleAsync( { await WriteStateAsync(cancellationToken); } - if (message.ThrowDuringPreparation || (message.ThrowOnceDuringPreparation && _failedOnce.Add(message.LogicalId))) + if (message.ThrowDuringPreparation || (message.ThrowOnceDuringPreparation && FailedOnce.TryAdd(message.LogicalId, 0))) { throw new InvalidOperationException($"Injected handler preparation failure for {message.LogicalId}."); } From dc64c00669445e675f2dab9ee03679ec1a54363b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 23:10:24 -0700 Subject: [PATCH 38/95] test(messaging): compare staged test outbox messages by command equivalence JournaledTestOutbox now coalesces a repeated send only when EnvelopeEquivalence.AreEquivalent holds, so DeliverAt participates in the comparison like the production outbox. --- test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs index 6109b7c0598..879ea8b4db9 100644 --- a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -34,10 +34,7 @@ public void Send(OutboxMessage envelope) } if (TryGetMessage(envelope.Envelope.MessageId, out var existing)) { - if (existing.ReceiverId != envelope.ReceiverId - || !string.Equals(TestApplicationProtocol.Subject(existing.Envelope), - TestApplicationProtocol.Subject(envelope.Envelope), StringComparison.Ordinal) - || !existing.Envelope.Payload.AsSpan().SequenceEqual(envelope.Envelope.Payload)) + if (!EnvelopeEquivalence.AreEquivalent(existing, envelope)) { throw new InvalidOperationException($"The outbox already contains a different envelope with message ID '{envelope.Envelope.MessageId}'."); } From 8a0d7be7afc14265d31425d8e41f67ca6aa41702 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 00:01:27 -0700 Subject: [PATCH 39/95] fix(messaging): bind IInbox to the delivered inbox AddMessaging replaces any earlier IInbox registration, because inbox delivery targets the concrete Inbox and handlers must register on that instance. --- .../Hosting/MessagingExtensions.cs | 2 +- .../PublicMessagingRegistrationTests.cs | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs index d0cd1978e82..8c7db451f7b 100644 --- a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -76,7 +76,7 @@ public static IServiceCollection AddMessaging(this IServiceCollection services, services.TryAddSingleton(); services.TryAddScoped(); services.TryAddScoped(); - services.TryAddScoped(static sp => sp.GetRequiredService()); + services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); services.TryAddScoped(); services.TryAddEnumerable( ServiceDescriptor.Singleton()); diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs index d076f34d609..6ae92eba69d 100644 --- a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -1,10 +1,13 @@ using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using Orleans.Hosting; using Orleans.Journaling; using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; using Orleans.Runtime; using Xunit; @@ -101,6 +104,22 @@ public void AddMessaging_InvalidOptionsFailThroughOptionsContract() Assert.Contains("DurableMessagingOptions validation failed", exception.Message, StringComparison.Ordinal); } + [Fact] + public async Task AddMessaging_BindsInboxContractToDeliveredInboxOverEarlierRegistration() + { + var services = new ServiceCollection(); + services.AddScoped(); + services.AddMessaging(); + + // Inbox construction needs an activation, so substitute an uninitialized scoped instance to observe the binding. + services.Replace(ServiceDescriptor.Scoped(static _ => (Inbox)RuntimeHelpers.GetUninitializedObject(typeof(Inbox)))); + await using var provider = services.BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IInbox)); + Assert.Same(scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService()); + } + [Fact] public async Task AddMessaging_DoesNotReplaceUnrelatedConstructionErrors() { From 0a3993fef41c03d50660425370892761956adad1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 10:22:50 -0700 Subject: [PATCH 40/95] feat(messaging): accept messages into application state through inbox sinks After validation and deduplication, the inbox offers each message without DeliverAt to the registered IInboxMessageSink instances. A sink which accepts it stages the message in its own journaled state, and the inbox journals only the processed record in the same write, so the payload is journaled once. A sink which throws dead-letters the message. Declined and scheduled messages follow the handler path. --- src/Orleans.Messaging/Inbox.cs | 139 +++++-- .../Functional/MessageSourceAndSinkTests.cs | 343 ++++++++++++++++++ .../Support/JournaledTestOutbox.cs | 6 + .../RecordingJournalStorageProvider.cs | 97 +++++ 4 files changed, 562 insertions(+), 23 deletions(-) create mode 100644 test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index f8c0fb43d1f..1f53c2179f6 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -25,6 +25,7 @@ internal sealed partial class Inbox : IInbox, IInboxExtension, IStateMachine, ID { private readonly MessagingPump _pump; private readonly InboxDispatcher _handlers = new([]); + private readonly List _sinks = []; public Inbox(MessagingPump pump) { @@ -53,6 +54,13 @@ public IInbox Register(IInboxHandler handler) return this; } + public IInbox Register(IInboxMessageSink sink) + { + ArgumentNullException.ThrowIfNull(sink); + _sinks.Add(sink); + return this; + } + public async ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -64,26 +72,22 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel var key = message.Envelope.MessageId; try { - if (_processed.TryGetValue(key, out var processedAt) - && !MessagingTime.IsExpired(TimeProvider.GetUtcNow(), processedAt, Options.DeduplicationWindow)) + if (IsDuplicate(message, out var pending)) { + // The sender may retire its copy once this returns, so the acceptance must be durable. A sink acceptance + // journals only its processed record, which can still be in flight. + if (pending is null ? _sinks.Count > 0 : !IsDurable(pending)) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + if (pending is not null) _pump.Signal(); + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); return DeliveryResult.Duplicate(); } - if (_messages.TryGetValue(key, out var pending)) + // Sink-accepted messages never occupy inbox capacity and need no handler. + if (message.Envelope.DeliverAt is null && StageSinkAcceptance(message) is { } accepted) { - if (!EnvelopeEquivalence.AreSameCommand(pending.Message, message)) - { - throw new InvalidOperationException( - $"The inbox already contains a different command with message ID '{key}'."); - } - - // The sender may retire its copy once this returns, so the acceptance must be durable. - if (!IsDurable(pending)) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); - _pump.Signal(); - Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); - return DeliveryResult.Duplicate(); + await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + return accepted; } // Scheduled messages never count toward capacity, so they are not backpressured either. @@ -113,6 +117,89 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel } } + /// + /// Offers a message to the registered sinks and stages the outcome of the first sink which accepts or rejects it. + /// + /// The delivery result, or when every sink declines the message. + /// + /// An accepted message journals only its processed record, in the write which commits the sink's staged changes. + /// A sink which throws stages nothing, so the message is dead-lettered with its payload and recorded as processed. + /// + private DeliveryResult? StageSinkAcceptance(InboxMessage message) + { + var key = message.Envelope.MessageId; + foreach (var sink in _sinks) + { + Exception? error = null; + try + { + if (!sink.TryAccept(message)) continue; + } + catch (Exception exception) + { + error = exception; + } + + try + { + if (error is null) + { + StageCompletion(key, deadLetter: null); + } + else + { + LogSinkRejected(Logger, error, key, GrainContext.GrainId); + StageDeadLetter(message, error.Message, attemptCount: 1); + } + } + catch (Exception exception) + { + // The sink's staged changes cannot commit without the processed record. + LogStagingFailed(Logger, exception); + _pump.Fail(exception, "Inbox staging failed."); + throw; + } + + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); + Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, error is null ? "success" : "dead_lettered"); + if (error is not null) return DeliveryResult.DeadLettered(error.Message); + LogMessageAccepted(Logger, key, GrainContext.GrainId); + return DeliveryResult.Accepted(); + } + + return null; + } + + /// + /// Returns whether the message was completed within the deduplication window or is pending, and rejects a pending + /// command which differs from it. + /// + /// The received message. + /// The pending entry, or when the message was completed or is new. + private bool IsDuplicate(InboxMessage message, out InboxEntry? pending) + { + var key = message.Envelope.MessageId; + if (_processed.TryGetValue(key, out var processedAt) + && !MessagingTime.IsExpired(TimeProvider.GetUtcNow(), processedAt, Options.DeduplicationWindow)) + { + pending = null; + return true; + } + + if (_messages.TryGetValue(key, out pending)) + { + if (!EnvelopeEquivalence.AreSameCommand(pending.Message, message)) + { + throw new InvalidOperationException( + $"The inbox already contains a different command with message ID '{key}'."); + } + + return true; + } + + return false; + } + /// /// Processes one batch of durable, due messages and returns whether any message was processed. /// @@ -204,7 +291,7 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell var attempts = checked(entry.State.AttemptCount + 1); if (attempts >= Options.MaxProcessingAttempts) { - StageDeadLetter(entry, error.Message, attempts); + StageDeadLetter(entry.Message, error.Message, attempts); status = "dead_lettered"; } else @@ -236,23 +323,23 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell private bool IsCurrent(InboxEntry entry) => _messages.TryGetValue(entry.Message.Envelope.MessageId, out var current) && ReferenceEquals(current, entry); - private void StageCompletion(InboxEntry entry, InboxDeadLetter? deadLetter) => + private void StageCompletion(HierarchicalKey messageId, InboxDeadLetter? deadLetter) => Emit(new() { Kind = InboxCommandKind.Complete, - MessageId = entry.Message.Envelope.MessageId, + MessageId = messageId, ProcessedAt = TimeProvider.GetUtcNow(), DeadLetter = deadLetter }); - private void StageDeadLetter(InboxEntry entry, string reason, int attemptCount) + private void StageDeadLetter(InboxMessage message, string reason, int attemptCount) { var now = TimeProvider.GetUtcNow(); - var key = entry.Message.Envelope.MessageId; + var key = message.Envelope.MessageId; CompactDeadLetters(now, reservedCapacity: _deadLetters.ContainsKey(key) ? 0 : 1); - StageCompletion(entry, new InboxDeadLetter + StageCompletion(key, new InboxDeadLetter { - Message = entry.Message, + Message = message, DeadLetteredAt = now, Reason = reason, AttemptCount = attemptCount @@ -383,7 +470,7 @@ public void Complete() try { - owner.StageCompletion(entry, deadLetter: null); + owner.StageCompletion(entry.Message.Envelope.MessageId, deadLetter: null); Completed = true; } catch (Exception exception) @@ -408,7 +495,7 @@ public void Fail(string reason) try { - owner.StageDeadLetter(entry, reason, checked(entry.State.AttemptCount + 1)); + owner.StageDeadLetter(entry.Message, reason, checked(entry.State.AttemptCount + 1)); DeadLetterReason = reason; Completed = true; } @@ -441,6 +528,9 @@ private void ValidateAttempt() Message = "Inbox delivery of message {MessageId} to {GrainId} failed")] private static partial void LogDeliveryOperationFailed(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId grainId); + [LoggerMessage(Level = LogLevel.Error, Message = "Inbox command staging failed.")] + private static partial void LogStagingFailed(ILogger logger, Exception exception); + [LoggerMessage(Level = LogLevel.Warning, Message = "No inbox handler registered for message {MessageId} to {ReceiverId}")] private static partial void LogHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId receiverId); @@ -449,4 +539,7 @@ private void ValidateAttempt() [LoggerMessage(Level = LogLevel.Error, Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] private static partial void LogHandlerException(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Inbox message sink rejected message {MessageId} to {ReceiverId}; the message is dead-lettered")] + private static partial void LogSinkRejected(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs new file mode 100644 index 00000000000..39905d646c2 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -0,0 +1,343 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Serialization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Journaling.Json; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Outbox sources and inbox sinks keep application-owned payloads as the only journaled copy. +/// +public abstract class MessageSourceAndSinkTests(MessageSourceAndSinkFixture fixture) +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + private InProcessTestCluster Cluster => fixture.Cluster; + private RecordingJournalStorageProvider Storage => fixture.Storage; + private SourceSinkProbe Probe => fixture.Probe; + + [Fact] + public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndIsOfferedAgainAfterItExpires() + { + var receiver = NewGrain(); + var message = NewInboxMessage("deduplicated", NewPayload()); + var extension = receiver.AsReference(); + + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, await OnGrainAsync(receiver, context => Inbox(context).Count)); + + fixture.Clock.Advance(MessageSourceAndSinkFixture.DeduplicationWindow); + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(2, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + Assert.Equal(0, await OnGrainAsync(receiver, context => Inbox(context).Count)); + } + + [Fact] + public async Task SinkException_DeadLettersTheMessageAndRecordsItAsProcessed() + { + var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.Throw; + var payload = NewPayload(); + var message = NewInboxMessage("rejected", payload); + var extension = receiver.AsReference(); + + var result = await extension.DeliverAsync(message, Token); + + Assert.Equal(DeliveryStatus.DeadLettered, result.Status); + Assert.Equal(SourceSinkGrain.Rejection, result.Message); + var (state, count, letters) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState(), + Inbox(context).Count, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Empty(state.Received); + Assert.Equal(0, count); + var letter = Assert.Single(letters); + Assert.Equal(payload, letter.Message.Envelope.Payload); + Assert.Equal(1, letter.AttemptCount); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + Probe.For(receiver).Mode = SinkMode.Accept; + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(0, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + } + + [Fact] + public async Task ScheduledAndDeclinedMessages_UseInboxAcceptanceAndHandlers() + { + var receiver = NewGrain(); + var extension = receiver.AsReference(); + var scheduled = NewInboxMessage("scheduled", NewPayload(), DateTimeOffset.UtcNow.AddDays(1)); + + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); + Assert.Equal(1, await OnGrainAsync(receiver, context => Inbox(context).Count)); + Assert.Equal(0, Probe.For(receiver).SinkOffers); + + Probe.For(receiver).Mode = SinkMode.Decline; + var declinedPayload = NewPayload(); + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(NewInboxMessage("declined", declinedPayload), Token)).Status); + await WaitUntilAsync(receiver, _ => Probe.For(receiver).Handled == 1); + Assert.Equal(1, Probe.For(receiver).SinkOffers); + Assert.Equal(0, Probe.For(receiver).SinkAcceptances); + Assert.Empty((await OnGrainAsync(receiver, context => Grain(context).GetState())).Received); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), declinedPayload)); + } + + private ISourceSinkGrain NewGrain() => Cluster.Client.GetGrain(Guid.NewGuid()); + + private static HierarchicalKey NewMessageId(string name) => HierarchicalKey.Create("source-sink", name, Guid.NewGuid().ToString("N")); + + private static byte[] NewPayload() => RandomNumberGenerator.GetBytes(48); + + private static InboxMessage NewInboxMessage(string name, byte[] payload, DateTimeOffset? deliverAt = null) => new() + { + Envelope = new Envelope { MessageId = NewMessageId(name), Payload = payload, Subject = SourceSinkGrain.Subject, DeliverAt = deliverAt } + }; + + private static JournalId Journal(IAddressable grain) => JournalId.FromGrainId(grain.GetGrainId()); + + private byte[][] Writes(IAddressable grain) => Storage.GetWrites(Journal(grain)); + + private static SourceSinkGrain Grain(IGrainContext context) => Assert.IsType(context.GrainInstance); + + private static Inbox Inbox(IGrainContext context) => context.ActivationServices.GetRequiredService(); + + private Task OnGrainAsync(IAddressable grain, Func func) + { + if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) + { + throw new InvalidOperationException($"Grain '{grain.GetGrainId()}' is not active."); + } + + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Scheduler.QueueAction(() => + { + try + { + done.SetResult(func(context)); + } + catch (Exception exception) + { + done.SetException(exception); + } + }); + return done.Task.WaitAsync(TimeSpan.FromSeconds(30), Token); + } + + private async Task WaitUntilAsync(IAddressable grain, Func predicate) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!Cluster.TryGetGrainContext(grain.GetGrainId(), out _) || !await OnGrainAsync(grain, predicate)) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Condition was not met for grain '{grain.GetGrainId()}'."); + } + + await Task.Delay(10, Token); + } + } +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JsonMessageSourceAndSinkTests(JsonMessageSourceAndSinkTests.Fixture fixture) + : MessageSourceAndSinkTests(fixture), IClassFixture +{ + public sealed class Fixture() : MessageSourceAndSinkFixture("json"); +} + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class BinaryMessageSourceAndSinkTests(BinaryMessageSourceAndSinkTests.Fixture fixture) + : MessageSourceAndSinkTests(fixture), IClassFixture +{ + public sealed class Fixture() : MessageSourceAndSinkFixture("orleans-binary"); +} + +public abstract class MessageSourceAndSinkFixture(string journalFormatKey) : IAsyncLifetime +{ + public static readonly TimeSpan DeduplicationWindow = TimeSpan.FromMinutes(10); + + public FakeTimeProvider Clock { get; } = new(DateTimeOffset.UtcNow); + public RecordingJournalStorageProvider Storage { get; } = new(); + public SourceSinkProbe Probe { get; } = new(); + public InProcessTestCluster Cluster { get; private set; } = null!; + + public async ValueTask InitializeAsync() + { + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureSilo((_, silo) => + { + silo.AddVolatileJournalStorage(); + silo.Services.Configure(options => options.JournalFormatKey = journalFormatKey); + silo.Services.Configure(options => options.AddTypeInfoResolver(SourceSinkJsonContext.Default)); + silo.Services.AddSingleton(Clock); + silo.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + silo.UseInMemoryDurableJobs(); + silo.AddMessaging(options => + { + options.DeduplicationWindow = DeduplicationWindow; + options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); + options.MaxDeliveryAttempts = 2; + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(10); + }); + silo.Services.AddSingleton(Probe); + silo.Services.RemoveAll(); + silo.Services.RemoveAll(); + silo.Services.AddSingleton(services => + { + Storage.Configure(services.GetRequiredService>()); + return Storage; + }); + silo.Services.AddSingleton( + services => (IJournalStorageCatalog)services.GetRequiredService()); + }); + Cluster = builder.Build(); + await Cluster.DeployAsync(TestContext.Current.CancellationToken); + } + + public async ValueTask DisposeAsync() => await Cluster.DisposeAsync(); +} + +[JsonSerializable(typeof(string))] +[JsonSerializable(typeof(byte[]))] +internal sealed partial class SourceSinkJsonContext : JsonSerializerContext; + +public enum SinkMode +{ + Accept, + Decline, + Throw +} + +public sealed class SourceSinkProbe +{ + private readonly ConcurrentDictionary _counters = new(); + + public Counters For(IAddressable grain) => For(grain.GetGrainId()); + + public Counters For(GrainId grainId) => _counters.GetOrAdd(grainId, static _ => new()); + + public sealed class Counters + { + private int _materializations; + private int _sinkOffers; + private int _sinkAcceptances; + private int _handled; + + public volatile SinkMode Mode; + public int Materializations => Volatile.Read(ref _materializations); + public int SinkOffers => Volatile.Read(ref _sinkOffers); + public int SinkAcceptances => Volatile.Read(ref _sinkAcceptances); + public int Handled => Volatile.Read(ref _handled); + + public void OnMaterialized() => Interlocked.Increment(ref _materializations); + public void OnSinkOffered() => Interlocked.Increment(ref _sinkOffers); + public void OnSinkAccepted() => Interlocked.Increment(ref _sinkAcceptances); + public void OnHandled() => Interlocked.Increment(ref _handled); + } +} + +public interface ISourceSinkGrain : IGrainWithGuidKey +{ + Task WriteAsync(); + Task DeactivateAsync(); +} + +/// Stores received payloads as an inbox sink. +public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, IInboxMessageSink +{ + internal const string Subject = "source-sink/payload"; + internal const string Rejection = "The sink rejected the message."; + private readonly IJournaledStateManager _manager; + private readonly IDurableDictionary _outgoing; + private readonly IDurableDictionary _received; + private readonly IDurableDictionary _retired; + private readonly SourceSinkProbe.Counters _probe; + + public SourceSinkGrain( + IGrainContext context, + IJournaledStateManager manager, + IInbox inbox, + [FromKeyedServices("outgoing")] IDurableDictionary outgoing, + [FromKeyedServices("received")] IDurableDictionary received, + [FromKeyedServices("retired")] IDurableDictionary retired, + SourceSinkProbe probe) + { + _manager = manager; + _outgoing = outgoing; + _received = received; + _retired = retired; + _probe = probe.For(context.GrainId); + inbox.Register(this).Register(new Handler(_probe)); + } + + internal (Dictionary Outgoing, Dictionary Received, Dictionary Retired) GetState() => + (new(_outgoing), new(_received), new(_retired)); + + internal void StagePayload(HierarchicalKey messageId, byte[] payload) => _outgoing[messageId.ToString()] = payload; + + public async Task WriteAsync() => await _manager.WriteStateAsync(CancellationToken.None); + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + bool IInboxMessageSink.TryAccept(InboxMessage message) + { + if (message.Envelope.Subject != Subject) return false; + _probe.OnSinkOffered(); + switch (_probe.Mode) + { + case SinkMode.Decline: + return false; + case SinkMode.Throw: + throw new InvalidOperationException(Rejection); + } + + _received[message.Envelope.MessageId.ToString()] = message.Envelope.Payload; + _probe.OnSinkAccepted(); + return true; + } + + private sealed class Handler(SourceSinkProbe.Counters probe) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) => context.Message.Envelope.Subject == Subject; + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + probe.OnHandled(); + context.Complete(); + return ValueTask.CompletedTask; + } + } +} + +public interface INoHandlerGrain : IGrainWithGuidKey +{ + Task PingAsync(); +} + +/// A messaging grain without handlers or sinks, whose inbox reports every delivery as handler-not-found. +public sealed class NoHandlerGrain : Grain, INoHandlerGrain, IMessagingGrain +{ + public Task PingAsync() => Task.CompletedTask; +} diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs index 879ea8b4db9..7c744c33bd8 100644 --- a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -48,6 +48,12 @@ public void Send(OutboxMessage envelope) StoredMessages.Add(envelope.Envelope.MessageId, envelope); } + public void SendFromSource(HierarchicalKey messageId, GrainId receiverId) => + throw new NotSupportedException("The test outbox stores complete envelopes only."); + + public IOutbox Register(IOutboxMessageSource source) => + throw new NotSupportedException("The test outbox stores complete envelopes only."); + public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => StoredMessages.TryGetValue(messageId, out envelope); public Task OnStart(CancellationToken cancellationToken) => Task.CompletedTask; diff --git a/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs new file mode 100644 index 00000000000..ada31bd7610 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs @@ -0,0 +1,97 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.Text; +using Microsoft.Extensions.Options; +using Orleans.Journaling; + +namespace Orleans.Messaging.Tests.Support; + +/// Stores journals in memory and records the bytes of every successful journal write. +public sealed class RecordingJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog +{ + private readonly ConcurrentDictionary> _writes = new(); + private VolatileJournalStorageProvider? _inner; + + public void Configure(IOptions options) => _inner ??= new VolatileJournalStorageProvider(options); + + public IJournalStorage CreateStorage(JournalId journalId) => + new RecordingJournalStorage(_writes.GetOrAdd(journalId, static _ => new()), Inner.CreateStorage(journalId)); + + public IAsyncEnumerable ListAsync( + JournalCatalogListOptions? options = null, + CancellationToken cancellationToken = default) => + Inner.ListAsync(options, cancellationToken); + + /// Gets the bytes of each successful write to a journal, in write order. + public byte[][] GetWrites(JournalId journalId) => _writes.TryGetValue(journalId, out var writes) ? [.. writes] : []; + + /// + /// Counts the occurrences of across a journal's writes, as raw bytes in the Orleans binary + /// format or as the base64 string which the JSON format writes for a byte array. + /// + public int CountPayloadOccurrences(JournalId journalId, byte[] payload) + { + var base64 = Encoding.UTF8.GetBytes(Convert.ToBase64String(payload)); + var count = 0; + foreach (var write in GetWrites(journalId)) + { + count += CountOccurrences(write, payload) + CountOccurrences(write, base64); + } + + return count; + } + + private static int CountOccurrences(ReadOnlySpan data, ReadOnlySpan value) + { + var count = 0; + for (var index = data.IndexOf(value); index >= 0; index = data.IndexOf(value)) + { + count++; + data = data[(index + value.Length)..]; + } + + return count; + } + + private VolatileJournalStorageProvider Inner => + _inner ?? throw new InvalidOperationException("The recording journal storage provider has not been configured."); + + private sealed class RecordingJournalStorage(ConcurrentQueue writes, IJournalStorage inner) : IJournalStorage + { + public bool IsCompactionRequested => inner.IsCompactionRequested; + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) => + inner.ReadAsync(consumer, cancellationToken); + + public ValueTask CreateIfNotExistsAsync( + IReadOnlyDictionary? metadata = null, + CancellationToken cancellationToken = default) => + inner.CreateIfNotExistsAsync(metadata, cancellationToken); + + public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) => + inner.GetMetadataAsync(cancellationToken); + + public ValueTask UpdateMetadataAsync( + IReadOnlyDictionary? set = null, + IEnumerable? remove = null, + string? expectedETag = null, + CancellationToken cancellationToken = default) => + inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + + public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var bytes = value.ToArray(); + await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + writes.Enqueue(bytes); + } + + public async ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) + { + var bytes = value.ToArray(); + await inner.AppendAsync(value, cancellationToken).ConfigureAwait(false); + writes.Enqueue(bytes); + } + + public ValueTask DeleteAsync(CancellationToken cancellationToken) => inner.DeleteAsync(cancellationToken); + } +} From 539cbd0a5411703ab799f31499319af8df439cf3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 11:16:14 -0700 Subject: [PATCH 41/95] test(messaging): implement the source-provided outbox send in the test outbox JournaledTestOutbox implements IOutbox.Send(HierarchicalKey), and the sink test grain drops its unused StagePayload helper. --- .../Functional/MessageSourceAndSinkTests.cs | 2 -- test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 39905d646c2..147f23b114f 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -291,8 +291,6 @@ public SourceSinkGrain( internal (Dictionary Outgoing, Dictionary Received, Dictionary Retired) GetState() => (new(_outgoing), new(_received), new(_retired)); - internal void StagePayload(HierarchicalKey messageId, byte[] payload) => _outgoing[messageId.ToString()] = payload; - public async Task WriteAsync() => await _manager.WriteStateAsync(CancellationToken.None); public Task DeactivateAsync() diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs index 7c744c33bd8..cfd135be49b 100644 --- a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -48,7 +48,7 @@ public void Send(OutboxMessage envelope) StoredMessages.Add(envelope.Envelope.MessageId, envelope); } - public void SendFromSource(HierarchicalKey messageId, GrainId receiverId) => + public void Send(HierarchicalKey messageId) => throw new NotSupportedException("The test outbox stores complete envelopes only."); public IOutbox Register(IOutboxMessageSource source) => From 03cf4e993978c808bfb86d458f520fd2c053b4fa Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 12:42:54 -0700 Subject: [PATCH 42/95] feat(messaging): identify messages with MessageId wrapping HierarchicalKey The inbox, its journaled state, inbox dead-letter diagnostics, and the inbox tests identify messages with MessageId. --- .../IMessagingDiagnostics.cs | 4 +- src/Orleans.Messaging/Inbox.State.cs | 12 +++--- src/Orleans.Messaging/Inbox.cs | 20 ++++----- .../Orleans.Messaging/Orleans.Messaging.cs | 2 +- .../MessagingCommandSerializationTests.cs | 10 ++--- .../Functional/DedupeExpiryBehaviorTests.cs | 2 +- .../Functional/InboxAdmissionTests.cs | 4 +- .../InboxEnvelopeValidationTests.cs | 2 +- .../Functional/InboxHandlerCompletionTests.cs | 10 ++--- .../Functional/InboxHeaderBehaviorTests.cs | 12 +++--- .../Functional/InboxIdentityTests.cs | 42 +++++++++---------- .../Functional/MessageSourceAndSinkTests.cs | 2 +- .../MessagingRoutingAndDeadLetterTests.cs | 2 +- .../ProcessedRetentionMaintenanceTests.cs | 2 +- .../Support/InboxTestAccess.cs | 16 +++---- .../Support/JournaledTestOutbox.cs | 12 +++--- .../Support/MessagingTestGrains.cs | 38 ++++++++--------- .../Support/NamedFactoryMessagingGrain.cs | 6 +-- .../Support/ReceiverTestServices.cs | 2 +- .../Support/TestApplicationProtocol.cs | 8 ++-- 20 files changed, 104 insertions(+), 104 deletions(-) diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs index b2c1e245e26..18c56eecb87 100644 --- a/src/Orleans.Messaging/IMessagingDiagnostics.cs +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -23,7 +23,7 @@ public interface IMessagingDiagnostics /// /// The removal becomes durable with the grain's next journal write. /// - bool RemoveInboxDeadLetter(HierarchicalKey messageId); + bool RemoveInboxDeadLetter(MessageId messageId); } /// @@ -73,5 +73,5 @@ internal sealed class MessagingDiagnostics(Inbox inbox) : IMessagingDiagnostics AttemptCount = entry.AttemptCount }).ToList(); - public bool RemoveInboxDeadLetter(HierarchicalKey messageId) => inbox.RemoveDeadLetter(messageId); + public bool RemoveInboxDeadLetter(MessageId messageId) => inbox.RemoveDeadLetter(messageId); } diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index 346e1ea7340..8c570960704 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -10,9 +10,9 @@ namespace Orleans.Messaging; internal sealed partial class Inbox { - private readonly Dictionary _messages = []; - private readonly Dictionary _processed = []; - private readonly Dictionary _deadLetters = []; + private readonly Dictionary _messages = []; + private readonly Dictionary _processed = []; + private readonly Dictionary _deadLetters = []; private JournalStreamWriter _writer; // Pending messages with a delivery time, which do not count toward MaxCapacity. @@ -170,11 +170,11 @@ internal sealed class InboxCommand { [Id(0)] public InboxCommandKind Kind { get; init; } [Id(1)] public InboxMessage? Message { get; init; } - [Id(2)] public HierarchicalKey MessageId { get; init; } + [Id(2)] public MessageId MessageId { get; init; } [Id(3)] public InboxMessageState? State { get; init; } [Id(4)] public DateTimeOffset ProcessedAt { get; init; } [Id(5)] public InboxDeadLetter? DeadLetter { get; init; } - [Id(6)] public HierarchicalKey[]? RemovedKeys { get; init; } + [Id(6)] public MessageId[]? RemovedKeys { get; init; } [Id(7)] public InboxSnapshot? Snapshot { get; init; } } @@ -189,7 +189,7 @@ internal sealed class InboxSnapshot [GenerateSerializer, Alias("Orleans.Messaging.InboxProcessedEntry")] internal sealed class InboxProcessedEntry { - [Id(0)] public HierarchicalKey MessageId { get; init; } + [Id(0)] public MessageId MessageId { get; init; } [Id(1)] public DateTimeOffset ProcessedAt { get; init; } } diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 1f53c2179f6..35ccd0d70f3 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -42,9 +42,9 @@ public Inbox(MessagingPump pump) internal int Count => _messages.Count; - internal IReadOnlyDictionary MessageState => _messages; + internal IReadOnlyDictionary MessageState => _messages; - internal IReadOnlyDictionary DeadLetterState => _deadLetters; + internal IReadOnlyDictionary DeadLetterState => _deadLetters; internal InboxDispatcher Handlers => _handlers; @@ -323,7 +323,7 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell private bool IsCurrent(InboxEntry entry) => _messages.TryGetValue(entry.Message.Envelope.MessageId, out var current) && ReferenceEquals(current, entry); - private void StageCompletion(HierarchicalKey messageId, InboxDeadLetter? deadLetter) => + private void StageCompletion(MessageId messageId, InboxDeadLetter? deadLetter) => Emit(new() { Kind = InboxCommandKind.Complete, @@ -349,7 +349,7 @@ private void StageDeadLetter(InboxMessage message, string reason, int attemptCou private bool CompactDeadLetters(DateTimeOffset now, int reservedCapacity = 0) { if (_deadLetters.Count == 0) return false; - var retained = new Dictionary(_deadLetters); + var retained = new Dictionary(_deadLetters); if (!DeadLetterRetention.Compact(retained, now, Options.DeadLetterRetentionPeriod, Options.MaxRetainedDeadLetters, static entry => entry.DeadLetteredAt, reservedCapacity)) { @@ -364,7 +364,7 @@ private bool CompactDeadLetters(DateTimeOffset now, int reservedCapacity = 0) return true; } - internal bool RemoveDeadLetter(HierarchicalKey messageId) + internal bool RemoveDeadLetter(MessageId messageId) { _pump.ShutdownToken.ThrowIfCancellationRequested(); if (!_deadLetters.ContainsKey(messageId)) return false; @@ -526,20 +526,20 @@ private void ValidateAttempt() [LoggerMessage(Level = LogLevel.Error, EventName = "DeliveryOperationFailed", Message = "Inbox delivery of message {MessageId} to {GrainId} failed")] - private static partial void LogDeliveryOperationFailed(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId grainId); + private static partial void LogDeliveryOperationFailed(ILogger logger, Exception exception, MessageId messageId, GrainId grainId); [LoggerMessage(Level = LogLevel.Error, Message = "Inbox command staging failed.")] private static partial void LogStagingFailed(ILogger logger, Exception exception); [LoggerMessage(Level = LogLevel.Warning, Message = "No inbox handler registered for message {MessageId} to {ReceiverId}")] - private static partial void LogHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + private static partial void LogHandlerNotFound(ILogger logger, MessageId messageId, GrainId receiverId); [LoggerMessage(Level = LogLevel.Information, Message = "Accepted message {MessageId} to {ReceiverId}")] - private static partial void LogMessageAccepted(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + private static partial void LogMessageAccepted(ILogger logger, MessageId messageId, GrainId receiverId); [LoggerMessage(Level = LogLevel.Error, Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] - private static partial void LogHandlerException(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + private static partial void LogHandlerException(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); [LoggerMessage(Level = LogLevel.Error, Message = "Inbox message sink rejected message {MessageId} to {ReceiverId}; the message is dead-lettered")] - private static partial void LogSinkRejected(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + private static partial void LogSinkRejected(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); } diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 2b5287cae4e..33376a57679 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -207,7 +207,7 @@ public partial interface IMessagingDiagnostics { System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } - bool RemoveInboxDeadLetter(HierarchicalKey messageId); + bool RemoveInboxDeadLetter(MessageId messageId); } public partial interface IMessagingGrain diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 6079c06a552..b63f3a36647 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -59,7 +59,7 @@ public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWh private static Envelope Envelope(string id, DateTimeOffset? deliverAt = null) => new() { - MessageId = HierarchicalKey.Create("orders", id), + MessageId = MessageId.Create("orders", id), Payload = [1, 2, 3], Subject = "orders.v1", Headers = new(StringComparer.Ordinal) { ["trace"] = [4, 5] }, @@ -73,16 +73,16 @@ private static InboxCommand[] CreateInboxCommands() [ new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("accept"), State = new() }, new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("scheduled", Now), State = new() }, - new() { Kind = InboxCommandKind.Retry, MessageId = HierarchicalKey.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom" } }, - new() { Kind = InboxCommandKind.Complete, MessageId = HierarchicalKey.Create("orders", "done"), ProcessedAt = Now, DeadLetter = deadLetter }, - new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [HierarchicalKey.Create("orders", "dead")] }, + new() { Kind = InboxCommandKind.Retry, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom" } }, + new() { Kind = InboxCommandKind.Complete, MessageId = MessageId.Create("orders", "done"), ProcessedAt = Now, DeadLetter = deadLetter }, + new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [MessageId.Create("orders", "dead")] }, new() { Kind = InboxCommandKind.Snapshot, Snapshot = new() { Messages = [new() { Message = InboxMessage("pending"), State = new() { AttemptCount = 1 } }, new() { Message = InboxMessage("scheduled", Now), State = new() }], - Processed = [new() { MessageId = HierarchicalKey.Create("orders", "done"), ProcessedAt = Now }], + Processed = [new() { MessageId = MessageId.Create("orders", "done"), ProcessedAt = Now }], DeadLetters = [deadLetter] } } diff --git a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs index 5f0336e18f3..af5bba1e8e3 100644 --- a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs @@ -92,7 +92,7 @@ private static async Task DeliverAsync( private InboxMessage CreateEnvelope( IMessagingTestGrain receiver, TestMessage message, - HierarchicalKey? messageId = null) + MessageId? messageId = null) { var sessions = fixture.Client.ServiceProvider.GetRequiredService(); var sender = GrainId.Create("expiry-test-sender", "stable"); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs index e8a6f82df80..d5550902070 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs @@ -113,7 +113,7 @@ public async Task LateOutgoingIntent_AfterCapture_WaitsForNextCapture() var context = Fixture.GetGrainContext(receiver); var outbox = (JournaledTestOutbox)context.ActivationServices.GetRequiredService(); var grain = Assert.IsType(context.GrainInstance); - var durable = context.ActivationServices.GetRequiredKeyedService>("test-handler-output"); + var durable = context.ActivationServices.GetRequiredKeyedService>("test-handler-output"); var sessions = Fixture.Client.ServiceProvider.GetRequiredService(); var first = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 1); var late = TestApplicationProtocol.Create(sessions, receiver.GetGrainId(), receiver.GetGrainId(), "messages/output", 2); @@ -204,7 +204,7 @@ public async Task MessagingPrimaryStates_AreTheRegisteredScopedDataInstances() Assert.Same(services.GetRequiredService(), inbox); Assert.True(manager.TryGetStateMachine("test-handler-output", out var output)); Assert.Same(((JournaledTestOutbox)services.GetRequiredService()).StoredMessages, output); - Assert.Same(services.GetRequiredKeyedService>("test-handler-output"), output); + Assert.Same(services.GetRequiredKeyedService>("test-handler-output"), output); } [Fact] diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs index c744619f9a1..fe0f26549d0 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs @@ -54,7 +54,7 @@ public async Task DirectEnvelopeWithExplicitValidIdentity_PreservesDeliveryAndDe var receiver = NewGrain(); var template = CreateEnvelope(receiver, NewMessage(187, "explicit-identity")); var envelope = TestApplicationProtocol.With(template, Sessions, - messageId: HierarchicalKey.Parse("tenant/test/command/explicit", null)); + messageId: MessageId.Parse("tenant/test/command/explicit", null)); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); Assert.Equal(1, Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Count); Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs index c6aecb8d773..70aef48bae0 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -556,7 +556,7 @@ private async Task CreateAsync() await receiver.GetSnapshotAsync(); var context = Fixture.GetGrainContext(receiver); var services = context.ActivationServices; - var effects = services.GetRequiredKeyedService>("test-effects"); + var effects = services.GetRequiredKeyedService>("test-effects"); var handler = new TestHandler(effects, context); var rig = new Rig(receiver, context, Assert.IsType(context.GrainInstance), services.GetRequiredService(), (JournaledTestOutbox)services.GetRequiredService(), @@ -675,16 +675,16 @@ async Task CompleteAsync() private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); private static Task WaitAsync(Task task) => task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, MessagingTestGrain Grain, - IJournaledStateManager Manager, JournaledTestOutbox Outbox, IDurableDictionary Effects, + IJournaledStateManager Manager, JournaledTestOutbox Outbox, IDurableDictionary Effects, TestHandler Handler, Inbox InboxState) { - public IReadOnlyDictionary Inbox => InboxTestAccess.GetPendingMessages(InboxState); - public IReadOnlyDictionary Processed => InboxTestAccess.GetProcessedMessages(InboxState); + public IReadOnlyDictionary Inbox => InboxTestAccess.GetPendingMessages(InboxState); + public IReadOnlyDictionary Processed => InboxTestAccess.GetProcessedMessages(InboxState); public JournalId Journal => JournalId.FromGrainId(Receiver.GetGrainId()); } - private sealed class TestHandler(IDurableDictionary effects, IGrainContext grainContext) : IInboxHandler, IDisposable + private sealed class TestHandler(IDurableDictionary effects, IGrainContext grainContext) : IInboxHandler, IDisposable { public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs index 6bc7c26a737..8cdccb8240b 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -27,7 +27,7 @@ public async Task RawMessage_AbsentOptionalMetadataCommitsAndDeduplicates(bool e var context = Fixture.GetGrainContext(receiver); var grain = Assert.IsType(context.GrainInstance); byte[] payload = empty ? [] : [0, 255, 128, 42]; - var message = Raw(HierarchicalKey.Create("raw", Guid.NewGuid().ToString("N")), payload); + var message = Raw(MessageId.Create("raw", Guid.NewGuid().ToString("N")), payload); Assert.Null(message.Envelope.Subject); Assert.Null(message.Envelope.Headers); using var handler = new RawHandler(context); @@ -61,7 +61,7 @@ public async Task PendingDuplicate_ChangedCustomHeadersPreservesOriginal() _ = await receiver.GetSnapshotAsync(); var context = Fixture.GetGrainContext(receiver); var grain = Assert.IsType(context.GrainInstance); - var key = HierarchicalKey.Create("custom", Guid.NewGuid().ToString("N")); + var key = MessageId.Create("custom", Guid.NewGuid().ToString("N")); var original = Raw(key, [1, 2, 3], "original"); var changed = Raw(key, [1, 2, 3], "changed"); using var handler = new RawHandler(context); @@ -95,7 +95,7 @@ await OnTurnAsync(context, async () => public async Task UnknownHeaders_RecoverAndRemainAvailableAfterRetryAndDeadLetter() { var receiver = NewGrain(); - var key = HierarchicalKey.Create("headers", Guid.NewGuid().ToString("N")); + var key = MessageId.Create("headers", Guid.NewGuid().ToString("N")); var body = NewMessage(810, "poison") with { LogicalId = key, ThrowDuringPreparation = true }; var application = CreateEnvelope(receiver, body); byte[] opaque = [0xff, 0, 0x80, 0x7f]; @@ -172,7 +172,7 @@ public async Task InvalidMessageHeader_RejectsBeforeSchedulingOrMutation(bool co var receiver = NewGrain(); _ = await receiver.GetSnapshotAsync(); var context = Fixture.GetGrainContext(receiver); - var key = HierarchicalKey.Create("malformed", Guid.NewGuid().ToString("N")); + var key = MessageId.Create("malformed", Guid.NewGuid().ToString("N")); var message = Raw(key, [1, 2]); if (completed) { @@ -200,7 +200,7 @@ await OnTurnAsync(context, async () => Assert.Empty(snapshot.Effects); } - private static InboxMessage Raw(HierarchicalKey key, byte[] payload, string? trace = null) + private static InboxMessage Raw(MessageId key, byte[] payload, string? trace = null) { Dictionary? headers = null; if (trace is not null) (headers ??= new(StringComparer.Ordinal))["custom.trace"] = Encoding.UTF8.GetBytes(trace); @@ -240,7 +240,7 @@ public async ValueTask HandleAsync(IInboxHandlerContext handler, CancellationTok Entered.TrySetResult(); await Release.Task.WaitAsync(cancellationToken); var key = handler.Message.Envelope.MessageId; - var effects = context.ActivationServices.GetRequiredKeyedService>("test-effects"); + var effects = context.ActivationServices.GetRequiredKeyedService>("test-effects"); effects[key] = new(key, 1, 800, "raw"); handler.Complete(); } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs index ba7f2eee8c5..3e1aebd0140 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -19,7 +19,7 @@ public async Task PendingCommand_NewSenderCoalescesAndPreservesOriginalEnvelope( { var rig = await CreateAsync(); using var handler = rig.Handler; - var key = HierarchicalKey.Create("tenant", "orders", "1042", "reserve"); + var key = MessageId.Create("tenant", "orders", "1042", "reserve"); var original = Create(rig, key, "inventory.reserve.v1", "original"); var repeated = TestApplicationProtocol.With(original, Sessions, sender: GrainId.Create("forwarder", "second")); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); @@ -57,7 +57,7 @@ public async Task PendingCommand_ConflictingContentRejectsBeforeSchedulingOrMuta { var rig = await CreateAsync(); using var handler = rig.Handler; - var key = HierarchicalKey.Create("tenant", "orders", "1043", "reserve"); + var key = MessageId.Create("tenant", "orders", "1043", "reserve"); var original = Create(rig, key, "inventory.reserve.v1", "original"); var conflictingSubject = difference switch { @@ -110,7 +110,7 @@ public async Task CompletedCommand_ChangedSenderSubjectAndBodyAcknowledgesWithou { var rig = await CreateAsync(); using var handler = rig.Handler; - var key = HierarchicalKey.Create("tenant", "orders", "1044", "reserve"); + var key = MessageId.Create("tenant", "orders", "1044", "reserve"); var original = Create(rig, key, "inventory.reserve.v1", "original"); var repeated = Create(rig, key, "completely.changed.subject.v2", "different-body", GrainId.Create("forwarder", "new-owner")); @@ -143,8 +143,8 @@ public async Task ExactParentChildAndSiblingCommandsHaveIndependentCompletions() var rig = await CreateAsync(); using var handler = rig.Handler; handler.Release.TrySetResult(); - var parent = HierarchicalKey.Create("tenant", "orders", "1045"); - var keys = new[] { parent, parent.CreateChildKey("reserve"), parent.CreateChildKey("release") }; + var parent = MessageId.Create("tenant", "orders", "1045"); + var keys = new[] { parent, new MessageId(parent.Key.CreateChildKey("reserve")), new MessageId(parent.Key.CreateChildKey("release")) }; for (var index = 0; index < keys.Length; index++) { var command = Create(rig, keys[index], "inventory.command.v1", $"command-{index}"); @@ -169,7 +169,7 @@ public async Task SameCommandKey_InDifferentReceiversCompletesIndependently() using var secondHandler = second.Handler; firstHandler.Release.TrySetResult(); secondHandler.Release.TrySetResult(); - var key = HierarchicalKey.Create("tenant", "orders", "1049", "reserve"); + var key = MessageId.Create("tenant", "orders", "1049", "reserve"); var one = Create(first, key, "inventory.reserve.v1", "first-receiver"); var two = Create(second, key, "inventory.reserve.v1", "second-receiver"); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(first.Receiver, one)).Status); @@ -191,7 +191,7 @@ public async Task PendingIdentityAfterReplayStillRejectsConflictAndDeduplicatesN { var rig = await CreateAsync(); using var handler = rig.Handler; - var key = HierarchicalKey.Create("tenant", "orders", "1046", "reserve"); + var key = MessageId.Create("tenant", "orders", "1046", "reserve"); var original = Create(rig, key, "inventory.reserve.v1", "original"); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(rig.Receiver, original)).Status); await WaitAsync(handler.Entered.Task); @@ -233,7 +233,7 @@ await OnTurnAsync(rig.Context, () => public async Task DeadLetterInspectionAndRemovalUseReceiverLocalKeyAcrossReplay() { var receiver = NewGrain(); - var key = HierarchicalKey.Create("tenant", "orders", "1048", "invalid-command"); + var key = MessageId.Create("tenant", "orders", "1048", "invalid-command"); var body = new TestMessage(key, 611, "poison", ThrowDuringPreparation: true); var original = CreateEnvelope(receiver, body); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, original)).Status); @@ -263,7 +263,7 @@ public async Task MissingOptionalMetadata_RawHandlerCompletesWithoutSubject(bool { var rig = await CreateAsync(); using var handler = rig.Handler; - var key = HierarchicalKey.Create("tenant", "invalid-subject", "command"); + var key = MessageId.Create("tenant", "invalid-subject", "command"); var envelope = Create(rig, key, "valid.subject.v1", "missing-subject"); var raw = TestApplicationProtocol.With(envelope, Sessions, removeSubject: true, removeSender: removeSender); var bytes = envelope.Envelope.Payload.ToArray(); @@ -290,9 +290,9 @@ public async Task OversizedMetadata_RejectsBeforeSchedulingOrPersistence(string using var handler = rig.Handler; var key = field switch { - "key-bytes" => HierarchicalKey.Create(new string('x', 1025)), - "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 513)), - _ => HierarchicalKey.Create("tenant", "bounded", "command") + "key-bytes" => MessageId.Create(new string('x', 1025)), + "key-utf8" => MessageId.Create(new string('\u00e9', 513)), + _ => MessageId.Create("tenant", "bounded", "command") }; var subject = field switch { @@ -325,10 +325,10 @@ public async Task MetadataAtExactDefaultLimit_IsAccepted(string field) handler.Release.TrySetResult(); var key = field switch { - "key-bytes" => HierarchicalKey.Create(new string('x', 1024)), - "key-utf8" => HierarchicalKey.Create(new string('\u00e9', 512)), - "key-depth" => HierarchicalKey.Create(Enumerable.Repeat("segment", 32).ToArray()), - _ => HierarchicalKey.Create("tenant", "bounded", "command") + "key-bytes" => MessageId.Create(new string('x', 1024)), + "key-utf8" => MessageId.Create(new string('\u00e9', 512)), + "key-depth" => MessageId.Create(Enumerable.Repeat("segment", 32).ToArray()), + _ => MessageId.Create("tenant", "bounded", "command") }; var subject = field switch { @@ -351,7 +351,7 @@ public async Task ExpiredIdentityCanAcceptChangedSenderSubjectAndBodyAtExactBoun var rig = await CreateAsync(); using var handler = rig.Handler; handler.Release.TrySetResult(); - var key = HierarchicalKey.Create("tenant", "orders", "1047", "reserve"); + var key = MessageId.Create("tenant", "orders", "1047", "reserve"); var original = Create(rig, key, "inventory.reserve.v1", "original"); var repeated = Create(rig, key, "inventory.reserve.v2", "changed", GrainId.Create("forwarder", "new")); @@ -384,7 +384,7 @@ private static Rig CreateRig(IMessagingTestGrain receiver, IGrainContext context context.ActivationServices.GetRequiredService(), context.ActivationServices.GetRequiredService()); - private InboxMessage Create(Rig rig, HierarchicalKey key, string subject, string value, GrainId sender = default) => + private InboxMessage Create(Rig rig, MessageId key, string subject, string value, GrainId sender = default) => TestApplicationProtocol.CreateInbox(Sessions, sender.IsDefault ? GrainId.Create("producer", "first") : sender, subject, new TestMessage(key, 610, value), key); @@ -431,8 +431,8 @@ async Task CompleteAsync() private sealed record Rig(IMessagingTestGrain Receiver, IGrainContext Context, MessagingTestGrain Grain, IdentityHandler Handler, IJournaledStateManager Manager, Inbox Inbox) { - public IReadOnlyDictionary Pending => InboxTestAccess.GetPendingMessages(Inbox); - public IReadOnlyDictionary Processed => InboxTestAccess.GetProcessedMessages(Inbox); + public IReadOnlyDictionary Pending => InboxTestAccess.GetPendingMessages(Inbox); + public IReadOnlyDictionary Processed => InboxTestAccess.GetProcessedMessages(Inbox); } private sealed class IdentityHandler : IInboxHandler, IDisposable @@ -445,7 +445,7 @@ private sealed class IdentityHandler : IInboxHandler, IDisposable public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { var services = ReceiverTestServices.CurrentGrainContext!.ActivationServices; - var effects = services.GetRequiredKeyedService>("test-effects"); + var effects = services.GetRequiredKeyedService>("test-effects"); var application = TestApplicationProtocol.Read(services.GetRequiredService(), context.Message); var body = Assert.IsType(application.Body); Calls++; diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 147f23b114f..4895dfce68a 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -98,7 +98,7 @@ public async Task ScheduledAndDeclinedMessages_UseInboxAcceptanceAndHandlers() private ISourceSinkGrain NewGrain() => Cluster.Client.GetGrain(Guid.NewGuid()); - private static HierarchicalKey NewMessageId(string name) => HierarchicalKey.Create("source-sink", name, Guid.NewGuid().ToString("N")); + private static MessageId NewMessageId(string name) => MessageId.Create("source-sink", name, Guid.NewGuid().ToString("N")); private static byte[] NewPayload() => RandomNumberGenerator.GetBytes(48); diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs index 098ddca7be1..cffdbb8193a 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingRoutingAndDeadLetterTests.cs @@ -18,7 +18,7 @@ public sealed class MessagingRoutingAndDeadLetterTests : MessagingBehaviorTestBa public async Task InboxDeadLettersRetainNewestEntriesWithinConfiguredCapacity() { var receiver = NewGrain(); - var messageIds = new List(); + var messageIds = new List(); for (var sequence = 0; sequence < 3; sequence++) { var envelope = CreateEnvelope( diff --git a/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs index 4b080a440a9..af8e02dba83 100644 --- a/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ProcessedRetentionMaintenanceTests.cs @@ -57,7 +57,7 @@ public async Task ExpiredProcessedId_IsAcceptedAgainWhileUnexpiredIdsDeduplicate await Fixture.WaitForEffectCountAsync(receiver, 2); } - private Task> GetProcessedAsync(IMessagingTestGrain receiver) + private Task> GetProcessedAsync(IMessagingTestGrain receiver) { var inbox = Fixture.GetInbox(receiver); return Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetProcessedKeys(inbox)); diff --git a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs index 0294304121d..eb75bbe6768 100644 --- a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs +++ b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs @@ -21,24 +21,24 @@ internal static class InboxTestAccess public static int GetProcessedCount(Inbox inbox) => GetProcessed(inbox).Count; - public static HashSet GetProcessedKeys(Inbox inbox) => [.. GetProcessed(inbox).Keys]; + public static HashSet GetProcessedKeys(Inbox inbox) => [.. GetProcessed(inbox).Keys]; - public static bool TryGetProcessedAt(Inbox inbox, HierarchicalKey messageId, out DateTimeOffset processedAt) => + public static bool TryGetProcessedAt(Inbox inbox, MessageId messageId, out DateTimeOffset processedAt) => GetProcessed(inbox).TryGetValue(messageId, out processedAt); - public static void SetProcessedAt(Inbox inbox, HierarchicalKey messageId, DateTimeOffset processedAt) => + public static void SetProcessedAt(Inbox inbox, MessageId messageId, DateTimeOffset processedAt) => GetProcessed(inbox)[messageId] = processedAt; - public static IReadOnlyDictionary GetPendingMessages(Inbox inbox) => inbox.MessageState; + public static IReadOnlyDictionary GetPendingMessages(Inbox inbox) => inbox.MessageState; - public static IReadOnlyDictionary GetProcessedMessages(Inbox inbox) => GetProcessed(inbox); + public static IReadOnlyDictionary GetProcessedMessages(Inbox inbox) => GetProcessed(inbox); - public static InboxMessageState GetMessageState(Inbox inbox, HierarchicalKey messageId) => + public static InboxMessageState GetMessageState(Inbox inbox, MessageId messageId) => inbox.MessageState[messageId].State; [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_messages")] - private static extern ref Dictionary GetMessages(Inbox inbox); + private static extern ref Dictionary GetMessages(Inbox inbox); [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_processed")] - private static extern ref Dictionary GetProcessed(Inbox inbox); + private static extern ref Dictionary GetProcessed(Inbox inbox); } diff --git a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs index cfd135be49b..fd14ff30bd9 100644 --- a/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs +++ b/test/Orleans.Messaging.Tests/Support/JournaledTestOutbox.cs @@ -9,18 +9,18 @@ namespace Orleans.Messaging.Tests.Support; // Captures opaque handler output in the same journal as inbox effects. Dispatch belongs to the outbox layer. -internal sealed class JournaledTestOutbox(IDurableDictionary messages) - : IOutbox, ILifecycleObserver, IEnumerable> +internal sealed class JournaledTestOutbox(IDurableDictionary messages) + : IOutbox, ILifecycleObserver, IEnumerable> { private bool _stopped; private readonly TaskCompletionSource _stopping = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task Stopping => _stopping.Task; public int SendCalls { get; private set; } public Exception? NextSendFailure { get; set; } - public IDurableDictionary StoredMessages { get; } = messages; + public IDurableDictionary StoredMessages { get; } = messages; public int Count => StoredMessages.Count; public IEnumerable Messages => StoredMessages.Values; - public IEnumerator> GetEnumerator() => StoredMessages.GetEnumerator(); + public IEnumerator> GetEnumerator() => StoredMessages.GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); public void Send(OutboxMessage envelope) @@ -48,13 +48,13 @@ public void Send(OutboxMessage envelope) StoredMessages.Add(envelope.Envelope.MessageId, envelope); } - public void Send(HierarchicalKey messageId) => + public void Send(MessageId messageId) => throw new NotSupportedException("The test outbox stores complete envelopes only."); public IOutbox Register(IOutboxMessageSource source) => throw new NotSupportedException("The test outbox stores complete envelopes only."); - public bool TryGetMessage(HierarchicalKey messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => + public bool TryGetMessage(MessageId messageId, [MaybeNullWhen(false)] out OutboxMessage envelope) => StoredMessages.TryGetValue(messageId, out envelope); public Task OnStart(CancellationToken cancellationToken) => Task.CompletedTask; public Task OnStop(CancellationToken cancellationToken) diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 1446ae3b8ab..fd2223133c4 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -13,16 +13,16 @@ namespace Orleans.Messaging.Tests.Support; public interface IMessagingTestGrain : IGrainWithGuidKey { - Task SendAsync(GrainId target, string route, TestMessage message); - Task SendDuplicateAsync(GrainId target, string route, TestMessage message); - Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message); - Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message); + Task SendAsync(GrainId target, string route, TestMessage message); + Task SendDuplicateAsync(GrainId target, string route, TestMessage message); + Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message); + Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message); Task RetryWriteStateAsync(); Task StageEffectAsync(MessageEffect effect); Task StageOutputAsync(OutboxMessage envelope); Task AcceptAndDeactivateAsync(InboxMessage envelope); Task ConfigureHandlerAsync(bool enabled); - Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId); + Task RemoveInboxDeadLetterAsync(MessageId messageId); Task GetSnapshotAsync(); Task RequestDeactivationAsync(); Task SetControlEnvelopeAsync(InboxMessage envelope); @@ -33,7 +33,7 @@ public interface IMessagingTestGrain : IGrainWithGuidKey [GenerateSerializer, Immutable] public sealed record TestMessage( - [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(0)] MessageId LogicalId, [property: Id(1)] int Sequence, [property: Id(2)] string Value, [property: Id(3)] GrainId? ForwardTo = null, @@ -44,7 +44,7 @@ public sealed record TestMessage( [GenerateSerializer, Immutable] public sealed record MessageEffect( - [property: Id(0)] HierarchicalKey LogicalId, + [property: Id(0)] MessageId LogicalId, [property: Id(1)] int Count, [property: Id(2)] int Sequence, [property: Id(3)] string Value); @@ -63,7 +63,7 @@ public sealed record EndpointSnapshot( [GenerateSerializer, Immutable] public sealed record DeadLetterSnapshot( - [property: Id(0)] HierarchicalKey MessageId, + [property: Id(0)] MessageId MessageId, [property: Id(1)] string Route, [property: Id(2)] string Reason, [property: Id(3)] int AttemptCount, @@ -78,7 +78,7 @@ public sealed class MessagingTestGrain : DurableGrain, IMessagingTestGrain, IDur private readonly IJournaledStateManager _journalOwner; private readonly IOutbox _outbox; private readonly IMessagingDiagnostics _diagnostics; - private readonly IDurableDictionary _effects; + private readonly IDurableDictionary _effects; private readonly Inbox _inboxState; private readonly IInbox _inbox; private readonly SerializerSessionPool _sessions; @@ -97,7 +97,7 @@ internal Exception? JournalPreparationFailure private int _activeHandlers; private int _maxConcurrentHandlers; // Spans activations so a replayed message does not fail again after recovery. - private static readonly ConcurrentDictionary FailedOnce = new(); + private static readonly ConcurrentDictionary FailedOnce = new(); public MessagingTestGrain( IGrainContext grainContext, @@ -105,7 +105,7 @@ public MessagingTestGrain( IInbox inbox, IOutbox outbox, IMessagingDiagnostics diagnostics, - [FromKeyedServices("test-effects")] IDurableDictionary effects, + [FromKeyedServices("test-effects")] IDurableDictionary effects, [FromKeyedServices("inbox")] IDurableValue applicationInboxState, @@ -144,7 +144,7 @@ public override async Task OnActivateAsync(CancellationToken cancellationToken) _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); } - public async Task SendAsync(GrainId target, string route, TestMessage message) + public async Task SendAsync(GrainId target, string route, TestMessage message) { var envelope = CreateEnvelope(target, route, message); _outbox.Send(envelope); @@ -152,7 +152,7 @@ public async Task SendAsync(GrainId target, string route, TestM return envelope.Envelope.MessageId; } - public async Task SendDuplicateAsync(GrainId target, string route, TestMessage message) + public async Task SendDuplicateAsync(GrainId target, string route, TestMessage message) { var envelope = CreateEnvelope(target, route, message); _outbox.Send(envelope); @@ -161,14 +161,14 @@ public async Task SendDuplicateAsync(GrainId target, string rou return envelope.Envelope.MessageId; } - public async Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message) + public async Task SendAndDeactivateAsync(GrainId target, string route, TestMessage message) { var messageId = await SendAsync(target, route, message); DeactivateOnIdle(); return messageId; } - public Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message) + public Task StageWithoutCommitAsync(GrainId target, string route, TestMessage message) { var envelope = CreateEnvelope(target, route, message); _outbox.Send(envelope); @@ -204,7 +204,7 @@ public Task ConfigureHandlerAsync(bool enabled) return Task.CompletedTask; } - public async Task RemoveInboxDeadLetterAsync(HierarchicalKey messageId) + public async Task RemoveInboxDeadLetterAsync(MessageId messageId) { if (!_diagnostics.RemoveInboxDeadLetter(messageId)) { @@ -323,9 +323,9 @@ public void Dispose() } private readonly ConcurrentQueue _captures = new(); - private readonly ConcurrentQueue _outputCaptures = new(); + private readonly ConcurrentQueue _outputCaptures = new(); internal IReadOnlyList Captures => _captures.ToArray(); - internal IReadOnlyList OutputCaptures => _outputCaptures.ToArray(); + internal IReadOnlyList OutputCaptures => _outputCaptures.ToArray(); internal Exception? NextApplyFailure { get; set; } internal TaskCompletionSource ApplyAttempted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -379,7 +379,7 @@ private async ValueTask HandleAsync( var outgoing = message.ForwardTo is { } destination ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), destination, "messages/forwarded", message with { ForwardTo = null, ThrowDuringPreparation = false }, - context.Message.Envelope.MessageId.CreateChildKey("forwarded")) + new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("forwarded"))) : (OutboxMessage?)null; if (NextApplyFailure is { } failure) { diff --git a/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs index b189eb93170..d54c3bb08c8 100644 --- a/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs +++ b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs @@ -43,7 +43,7 @@ public sealed class NamedFactoryMessagingGrain : Grain, INamedFactoryMessagingGr { private readonly IJournaledStateManager _owner; private readonly IOutbox _outbox; - private readonly IDurableDictionary _effects; + private readonly IDurableDictionary _effects; private readonly SerializerSessionPool _sessions; private readonly NamedFactoryMessagingProbe _probe; private readonly Guid _activationId = Guid.NewGuid(); @@ -52,7 +52,7 @@ public NamedFactoryMessagingGrain( IJournaledStateManager owner, IInbox inbox, IOutbox outbox, - [FromKeyedServices("cutover-effects")] IDurableDictionary effects, + [FromKeyedServices("cutover-effects")] IDurableDictionary effects, SerializerSessionPool sessions, NamedFactoryMessagingProbe probe) { @@ -92,7 +92,7 @@ public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationTok var message = Assert.IsType(TestApplicationProtocol.Read(_sessions, context.Message).Body); var outgoing = message.ForwardTo is { } target ? TestApplicationProtocol.Create(_sessions, this.GetGrainId(), target, "messages/forwarded", - message with { ForwardTo = null }, context.Message.Envelope.MessageId.CreateChildKey("forwarded")) + message with { ForwardTo = null }, new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("forwarded"))) : (OutboxMessage?)null; cancellationToken.ThrowIfCancellationRequested(); diff --git a/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs index 083ce7a1fb2..99eb91692e2 100644 --- a/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs +++ b/test/Orleans.Messaging.Tests/Support/ReceiverTestServices.cs @@ -22,6 +22,6 @@ public static void Add(IServiceCollection services, Action(); services.AddScoped(static sp => new JournaledTestOutbox( - sp.GetRequiredKeyedService>("test-handler-output"))); + sp.GetRequiredKeyedService>("test-handler-output"))); } } diff --git a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs index c0f4cb4d52c..90927f17c60 100644 --- a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs +++ b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs @@ -16,10 +16,10 @@ public sealed record TestApplicationMessage( internal static class TestApplicationProtocol { - public static HierarchicalKey NewMessageId() => HierarchicalKey.Create("tests", Guid.NewGuid().ToString("N")); + public static MessageId NewMessageId() => MessageId.Create("tests", Guid.NewGuid().ToString("N")); public static OutboxMessage Create(SerializerSessionPool sessions, GrainId sender, GrainId receiver, - string route, object? body, HierarchicalKey messageId = default) => + string route, object? body, MessageId messageId = default) => new() { ReceiverId = receiver, @@ -27,7 +27,7 @@ public static OutboxMessage Create(SerializerSessionPool sessions, GrainId sende }; public static InboxMessage CreateInbox(SerializerSessionPool sessions, GrainId sender, - string route, object? body, HierarchicalKey messageId = default) + string route, object? body, MessageId messageId = default) { var key = messageId.IsDefault ? body is TestMessage message ? message.LogicalId : NewMessageId() @@ -79,7 +79,7 @@ public static GrainId Sender(SerializerSessionPool sessions, Envelope envelope) envelope.Subject; public static InboxMessage With(InboxMessage message, SerializerSessionPool sessions, - HierarchicalKey? messageId = null, GrainId? sender = null, string? subject = null, + MessageId? messageId = null, GrainId? sender = null, string? subject = null, byte[]? payload = null, bool removeSender = false, bool removeSubject = false) { var headers = message.Envelope.Headers is { } existing From cc055de19833fa69317452edbc46a878c460a082 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 15:41:04 -0700 Subject: [PATCH 43/95] feat(messaging): offer scheduled messages to inbox sinks and let sinks report delivery results The inbox offers every validated, deduplicated message, including scheduled ones, to its sinks. The first sink which takes a message reports its DeliveryResult: Accepted stages the processed record, Duplicate waits for any write in progress, Backpressured stages nothing, and DeadLettered, a throw, or an unsupported status dead-letters the message. --- src/Orleans.Messaging/IInboxMessageSink.cs | 2 +- src/Orleans.Messaging/Inbox.cs | 44 ++++--- src/Orleans.Messaging/README.md | 25 ++-- .../Functional/MessageSourceAndSinkTests.cs | 111 +++++++++++++++--- 4 files changed, 143 insertions(+), 39 deletions(-) diff --git a/src/Orleans.Messaging/IInboxMessageSink.cs b/src/Orleans.Messaging/IInboxMessageSink.cs index 02fce8bb42e..27087c3db15 100644 --- a/src/Orleans.Messaging/IInboxMessageSink.cs +++ b/src/Orleans.Messaging/IInboxMessageSink.cs @@ -11,7 +11,7 @@ namespace Orleans.Messaging; /// has synchronously staged the message's effects in its own journaled state. The /// inbox then journals only the message's processed record, without its payload, in the same journal write, and reports /// the result once that write is durable. Run effects which depend on the write, such as starting work, from -/// Orleans.Journaling.IStateMachine.OnWriteCompleted. +/// . /// /// /// A sink-owned message never occupies inbox capacity and has no inbox retry lifecycle: the sink owns its processing, diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 35ccd0d70f3..8a949fafae6 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -83,11 +83,12 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel return DeliveryResult.Duplicate(); } - // Sink-accepted messages never occupy inbox capacity and need no handler. - if (message.Envelope.DeliverAt is null && StageSinkAcceptance(message) is { } accepted) + // Sink-owned messages never occupy inbox capacity and need no handler. A sink's duplicate can still be staged, + // so every outcome except backpressure is reported once the write is durable. + if (StageSinkAcceptance(message) is { } sinkResult) { - await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); - return accepted; + if (sinkResult.Status != DeliveryStatus.Backpressured) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + return sinkResult; } // Scheduled messages never count toward capacity, so they are not backpressured either. @@ -118,38 +119,53 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel } /// - /// Offers a message to the registered sinks and stages the outcome of the first sink which accepts or rejects it. + /// Offers a message to the registered sinks and stages the outcome which the first sink to take the message reports. /// /// The delivery result, or when every sink declines the message. /// /// An accepted message journals only its processed record, in the write which commits the sink's staged changes. - /// A sink which throws stages nothing, so the message is dead-lettered with its payload and recorded as processed. + /// A duplicate or backpressured message stages nothing. A sink which dead-letters the message, throws, or reports + /// another status stages nothing itself, so the message is dead-lettered with its payload and recorded as processed. /// private DeliveryResult? StageSinkAcceptance(InboxMessage message) { var key = message.Envelope.MessageId; foreach (var sink in _sinks) { + DeliveryResult result; Exception? error = null; try { - if (!sink.TryAccept(message)) continue; + if (!sink.TryAccept(message, out result)) continue; + if (result.Status is not (DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.Backpressured or DeliveryStatus.DeadLettered)) + { + throw new InvalidOperationException($"The inbox message sink reported the unsupported delivery status '{result.Status}'."); + } } catch (Exception exception) { error = exception; + result = DeliveryResult.DeadLettered(exception.Message); + } + + if (result.Status is DeliveryStatus.Duplicate or DeliveryStatus.Backpressured) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, result.Status == DeliveryStatus.Duplicate ? "duplicate" : "backpressured"); + return result; } + var accepted = result.Status == DeliveryStatus.Accepted; + var reason = result.Message ?? "The inbox message sink dead-lettered the message."; try { - if (error is null) + if (accepted) { StageCompletion(key, deadLetter: null); } else { - LogSinkRejected(Logger, error, key, GrainContext.GrainId); - StageDeadLetter(message, error.Message, attemptCount: 1); + LogSinkDeadLettered(Logger, error, key, GrainContext.GrainId, reason); + StageDeadLetter(message, reason, attemptCount: 1); } } catch (Exception exception) @@ -161,8 +177,8 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel } Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); - Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, error is null ? "success" : "dead_lettered"); - if (error is not null) return DeliveryResult.DeadLettered(error.Message); + Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, accepted ? "success" : "dead_lettered"); + if (!accepted) return DeliveryResult.DeadLettered(reason); LogMessageAccepted(Logger, key, GrainContext.GrainId); return DeliveryResult.Accepted(); } @@ -540,6 +556,6 @@ private void ValidateAttempt() [LoggerMessage(Level = LogLevel.Error, Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] private static partial void LogHandlerException(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); - [LoggerMessage(Level = LogLevel.Error, Message = "Inbox message sink rejected message {MessageId} to {ReceiverId}; the message is dead-lettered")] - private static partial void LogSinkRejected(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); + [LoggerMessage(Level = LogLevel.Error, Message = "Inbox message sink dead-lettered message {MessageId} to {ReceiverId}: {Reason}")] + private static partial void LogSinkDeadLettered(ILogger logger, Exception? exception, MessageId messageId, GrainId receiverId, string reason); } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index d4bd00ba98c..492450cbcf7 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -103,16 +103,23 @@ construction or activation setup, before journal recovery. source that throws or returns an invalid message dead-letters that entry alone, with `OnRetired(DeadLettered)` in the same journal write; an exception during `Send` propagates without staging anything. -- `IInbox.Register(IInboxMessageSink)` adds a sink. The inbox offers each message, - including self-sent and scheduled messages, to the sinks in order. The first sink - whose `TryAccept` returns `true` reports the `DeliveryResult`: - - `Accepted`: the sink staged the message in its own journaled state. +- `IInbox.Register(IInboxMessageSink)` adds a sink. After validation and + deduplication, the inbox offers each message, including self-sent and scheduled + messages, to the sinks in order. The first sink whose `TryAccept` returns `true` + reports the `DeliveryResult`: + - `Accepted`: the sink staged the message in its own state. The inbox journals + only the processed record in the same write and returns `Accepted` once it is + durable. - `Duplicate`: the sink already holds the message. - - `Backpressured`: the sink is at capacity. A self-sent message fails its `Send`. - - `DeadLettered`: the sink rejects the message with a reason. A sink that throws - or reports another status dead-letters the message. - - A sink that takes a scheduled message honors its `DeliverAt`. Sinks are idempotent. + - `Backpressured`: the sink is at capacity, so the sender retries. A self-sent + message fails its `Send` instead. + - `DeadLettered`: the inbox dead-letters the message with the sink's reason. A + sink that throws or reports another status is treated the same way. + + Sink-owned messages never occupy capacity and have no inbox retry lifecycle; the + sink honors their `DeliverAt`. Declined messages follow the handler path once + `DeliverAt` passes. A message is offered again after its processed record leaves + `DeduplicationWindow`, so sinks must be idempotent. ## Delivery and processing diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 4895dfce68a..9bef196de4e 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -27,7 +27,7 @@ public abstract class MessageSourceAndSinkTests(MessageSourceAndSinkFixture fixt private SourceSinkProbe Probe => fixture.Probe; [Fact] - public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndIsOfferedAgainAfterItExpires() + public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndTheSinkReportsLaterDuplicates() { var receiver = NewGrain(); var message = NewInboxMessage("deduplicated", NewPayload()); @@ -35,21 +35,26 @@ public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndIsOfferedAgainAft Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(1, Probe.For(receiver).SinkOffers); Assert.Equal(1, Probe.For(receiver).SinkAcceptances); Assert.Equal(0, await OnGrainAsync(receiver, context => Inbox(context).Count)); fixture.Clock.Advance(MessageSourceAndSinkFixture.DeduplicationWindow); - Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); - Assert.Equal(2, Probe.For(receiver).SinkAcceptances); + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(2, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); Assert.Equal(0, Probe.For(receiver).Handled); Assert.Equal(0, await OnGrainAsync(receiver, context => Inbox(context).Count)); } - [Fact] - public async Task SinkException_DeadLettersTheMessageAndRecordsItAsProcessed() + [Theory] + [InlineData(SinkMode.Throw, SourceSinkGrain.Rejection)] + [InlineData(SinkMode.DeadLetter, SourceSinkGrain.Rejection)] + [InlineData(SinkMode.UnsupportedStatus, "The inbox message sink reported the unsupported delivery status 'HandlerNotFound'.")] + public async Task SinkRejection_DeadLettersTheMessageAndRecordsItAsProcessed(SinkMode mode, string reason) { var receiver = NewGrain(); - Probe.For(receiver).Mode = SinkMode.Throw; + Probe.For(receiver).Mode = mode; var payload = NewPayload(); var message = NewInboxMessage("rejected", payload); var extension = receiver.AsReference(); @@ -57,7 +62,7 @@ public async Task SinkException_DeadLettersTheMessageAndRecordsItAsProcessed() var result = await extension.DeliverAsync(message, Token); Assert.Equal(DeliveryStatus.DeadLettered, result.Status); - Assert.Equal(SourceSinkGrain.Rejection, result.Message); + Assert.Equal(reason, result.Message); var (state, count, letters) = await OnGrainAsync(receiver, context => ( Grain(context).GetState(), Inbox(context).Count, @@ -66,6 +71,7 @@ public async Task SinkException_DeadLettersTheMessageAndRecordsItAsProcessed() Assert.Equal(0, count); var letter = Assert.Single(letters); Assert.Equal(payload, letter.Message.Envelope.Payload); + Assert.Equal(reason, letter.Reason); Assert.Equal(1, letter.AttemptCount); Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); @@ -76,23 +82,64 @@ public async Task SinkException_DeadLettersTheMessageAndRecordsItAsProcessed() } [Fact] - public async Task ScheduledAndDeclinedMessages_UseInboxAcceptanceAndHandlers() + public async Task SinkBackpressure_StagesNothingAndTheRetryIsAccepted() { var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.Backpressure; + var payload = NewPayload(); + var message = NewInboxMessage("backpressured", payload); var extension = receiver.AsReference(); - var scheduled = NewInboxMessage("scheduled", NewPayload(), DateTimeOffset.UtcNow.AddDays(1)); + + Assert.Equal(DeliveryStatus.Backpressured, (await extension.DeliverAsync(message, Token)).Status); + var (count, letters) = await OnGrainAsync(receiver, context => ( + Inbox(context).Count, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Equal(0, count); + Assert.Empty(letters); + Assert.Equal(0, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + Probe.For(receiver).Mode = SinkMode.Accept; + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(2, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + } + + [Fact] + public async Task ScheduledMessage_IsOfferedToTheSinkBeforeItIsDue() + { + var receiver = NewGrain(); + var extension = receiver.AsReference(); + var payload = NewPayload(); + var scheduled = NewInboxMessage("scheduled", payload, DateTimeOffset.UtcNow.AddDays(1)); Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); - Assert.Equal(1, await OnGrainAsync(receiver, context => Inbox(context).Count)); - Assert.Equal(0, Probe.For(receiver).SinkOffers); + var (state, count) = await OnGrainAsync(receiver, context => (Grain(context).GetState(), Inbox(context).Count)); + Assert.Equal(payload, state.Received[scheduled.Envelope.MessageId.ToString()]); + Assert.Equal(0, count); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + } + + [Fact] + public async Task DeclinedMessages_UseInboxAcceptanceAndHandlers() + { + var receiver = NewGrain(); + var extension = receiver.AsReference(); Probe.For(receiver).Mode = SinkMode.Decline; + + var scheduled = NewInboxMessage("scheduled", NewPayload(), DateTimeOffset.UtcNow.AddDays(1)); + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); + Assert.Equal(1, await OnGrainAsync(receiver, context => Inbox(context).Count)); + var declinedPayload = NewPayload(); Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(NewInboxMessage("declined", declinedPayload), Token)).Status); await WaitUntilAsync(receiver, _ => Probe.For(receiver).Handled == 1); - Assert.Equal(1, Probe.For(receiver).SinkOffers); + Assert.Equal(2, Probe.For(receiver).SinkOffers); Assert.Equal(0, Probe.For(receiver).SinkAcceptances); Assert.Empty((await OnGrainAsync(receiver, context => Grain(context).GetState())).Received); + Assert.Equal(1, await OnGrainAsync(receiver, context => Inbox(context).Count)); Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), declinedPayload)); } @@ -223,7 +270,18 @@ public enum SinkMode { Accept, Decline, - Throw + Throw, + DeadLetter, + Backpressure, + UnsupportedStatus +} + +/// How the source fails to provide a message which it stores. +public enum SourceFault +{ + Throw, + InvalidMessage, + MismatchedId } public sealed class SourceSinkProbe @@ -236,17 +294,21 @@ public sealed class SourceSinkProbe public sealed class Counters { + private int _activations; private int _materializations; private int _sinkOffers; private int _sinkAcceptances; private int _handled; public volatile SinkMode Mode; + public ConcurrentDictionary Faults { get; } = new(); + public int Activations => Volatile.Read(ref _activations); public int Materializations => Volatile.Read(ref _materializations); public int SinkOffers => Volatile.Read(ref _sinkOffers); public int SinkAcceptances => Volatile.Read(ref _sinkAcceptances); public int Handled => Volatile.Read(ref _handled); + public void OnActivated() => Interlocked.Increment(ref _activations); public void OnMaterialized() => Interlocked.Increment(ref _materializations); public void OnSinkOffered() => Interlocked.Increment(ref _sinkOffers); public void OnSinkAccepted() => Interlocked.Increment(ref _sinkAcceptances); @@ -285,6 +347,7 @@ public SourceSinkGrain( _received = received; _retired = retired; _probe = probe.For(context.GrainId); + _probe.OnActivated(); inbox.Register(this).Register(new Handler(_probe)); } @@ -299,20 +362,38 @@ public Task DeactivateAsync() return Task.CompletedTask; } - bool IInboxMessageSink.TryAccept(InboxMessage message) + bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result) { + result = default; if (message.Envelope.Subject != Subject) return false; _probe.OnSinkOffered(); + var key = message.Envelope.MessageId.ToString(); switch (_probe.Mode) { case SinkMode.Decline: return false; case SinkMode.Throw: throw new InvalidOperationException(Rejection); + case SinkMode.DeadLetter: + result = DeliveryResult.DeadLettered(Rejection); + return true; + case SinkMode.Backpressure: + result = DeliveryResult.Backpressured(); + return true; + case SinkMode.UnsupportedStatus: + result = DeliveryResult.HandlerNotFound(); + return true; + } + + if (_received.ContainsKey(key)) + { + result = DeliveryResult.Duplicate(); + return true; } - _received[message.Envelope.MessageId.ToString()] = message.Envelope.Payload; + _received[key] = message.Envelope.Payload; _probe.OnSinkAccepted(); + result = DeliveryResult.Accepted(); return true; } From 84147066e5b783dbbe3abecd43257588766be2a7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 15:45:06 -0700 Subject: [PATCH 44/95] refactor(messaging): rename IInboxExtension to IMessagingExtension --- src/Orleans.Messaging/Inbox.cs | 2 +- .../MessagingGrainTypeConfigurator.cs | 2 +- src/Orleans.Messaging/README.md | 4 ++-- src/api/Orleans.Messaging/Orleans.Messaging.cs | 14 +++++++------- .../Functional/DedupeExpiryBehaviorTests.cs | 2 +- .../Functional/InboxAdmissionTests.cs | 4 ++-- .../Functional/InboxCapacityBehaviorTests.cs | 2 +- .../Functional/InboxEnvelopeValidationTests.cs | 2 +- .../Functional/InboxHeaderBehaviorTests.cs | 4 ++-- .../Functional/InboxIdentityTests.cs | 2 +- .../Functional/MessageSourceAndSinkTests.cs | 10 +++++----- .../Functional/MessagingMetricCardinalityTests.cs | 2 +- .../Hosting/PublicMessagingRegistrationTests.cs | 2 +- .../Support/MessagingBehaviorTestBase.cs | 2 +- 14 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 8a949fafae6..85d31eb3413 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -21,7 +21,7 @@ namespace Orleans.Messaging; /// Accepted messages are processed only after their acceptance is durable and their , /// if any, has passed. /// -internal sealed partial class Inbox : IInbox, IInboxExtension, IStateMachine, IDurableValueCommandHandler +internal sealed partial class Inbox : IInbox, IMessagingExtension, IStateMachine, IDurableValueCommandHandler { private readonly MessagingPump _pump; private readonly InboxDispatcher _handlers = new([]); diff --git a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs index b34ae02031c..4f0d23c43fd 100644 --- a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs +++ b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs @@ -20,7 +20,7 @@ public void Configure(GrainType grainType, GrainProperties properties, GrainType MessagingActivationValidator.Validate(grainClass, placementStrategy); var activationServices = context.ActivationServices; var inbox = activationServices.GetRequiredService(); - context.GetComponent()!.GetOrSetExtension(() => inbox); + context.GetComponent()!.GetOrSetExtension(() => inbox); }); } } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 492450cbcf7..06b514a0d95 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -15,7 +15,7 @@ validated at startup. Register typed subjects with `AddMessageType(subject)`. Messaging grains derive from `DurableGrain` or implement `IMessagingGrain`, and constructor-inject `IInbox`. During activation, the messaging grain-type -configurator installs the grain's inbox extension. Stateless worker grains are +configurator installs the grain's messaging extension. Stateless worker grains are rejected; reentrant and interleaving grains are supported, and the grain's code is responsible for correctness when calls interleave. @@ -123,7 +123,7 @@ construction or activation setup, before journal recovery. ## Delivery and processing -`IInboxExtension.DeliverAsync` delivers a message to a grain's inbox. Accepted inbox +`IMessagingExtension.DeliverAsync` delivers a message to a grain's inbox. Accepted inbox messages are processed only after the receiver's journal write completes. Each receiver returns a `DeliveryResult`: diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 33376a57679..7e389e2c18e 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -196,13 +196,6 @@ public partial interface IInboxMessageSink bool TryAccept(InboxMessage message, out DeliveryResult result); } - [Alias("IMessagingExtension")] - public partial interface IMessagingExtension : Runtime.IGrainExtension, Runtime.IAddressable - { - [Alias("DeliverAsync")] - System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); - } - public partial interface IMessagingDiagnostics { System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } @@ -210,6 +203,13 @@ public partial interface IMessagingDiagnostics bool RemoveInboxDeadLetter(MessageId messageId); } + [Alias("IMessagingExtension")] + public partial interface IMessagingExtension : Runtime.IGrainExtension, Runtime.IAddressable + { + [Alias("DeliverAsync")] + System.Threading.Tasks.ValueTask DeliverAsync(InboxMessage message, System.Threading.CancellationToken cancellationToken = default); + } + public partial interface IMessagingGrain { } diff --git a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs index af5bba1e8e3..f4ba2b2d871 100644 --- a/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/DedupeExpiryBehaviorTests.cs @@ -87,7 +87,7 @@ private Task WaitForIdleInboxAsync(IMessagingTestGrain receive private static async Task DeliverAsync( IMessagingTestGrain receiver, InboxMessage envelope) => - await receiver.AsReference().DeliverAsync(envelope); + await receiver.AsReference().DeliverAsync(envelope); private InboxMessage CreateEnvelope( IMessagingTestGrain receiver, diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs index d5550902070..ed14cb9c708 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxAdmissionTests.cs @@ -65,7 +65,7 @@ await OnTurnAsync(attempt.Context, () => public async Task HandlerStorageFailure_DeactivatesAndFailsQueuedWritesAfterPriorAcknowledgement() { using var attempt = await PrepareAttemptAsync("fault-ordering"); - IInboxExtension extension = attempt.Context.ActivationServices.GetRequiredService(); + IMessagingExtension extension = attempt.Context.ActivationServices.GetRequiredService(); await OnTurnAsync(attempt.Context, () => { attempt.Context.ActivationServices.GetRequiredKeyedService>("inbox").Value = "prior-captured"; @@ -148,7 +148,7 @@ public async Task AcceptanceAfterCapture_IsAcknowledgedByLaterOperation() // The pump runs alongside grain calls, so holding the seed's handler keeps it pending in the inbox. using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/cutoff"); - IInboxExtension extension = context.ActivationServices.GetRequiredService(); + IMessagingExtension extension = context.ActivationServices.GetRequiredService(); Task initial = null!; await OnTurnAsync(context, () => initial = extension.DeliverAsync(seed, TestContext.Current.CancellationToken).AsTask()); Assert.Equal(DeliveryStatus.Accepted, (await initial).Status); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs index b3d66a2d516..ddc17b88638 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxCapacityBehaviorTests.cs @@ -76,5 +76,5 @@ await fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), private static async Task DeliverAsync( IMessagingTestGrain receiver, InboxMessage envelope) => - await receiver.AsReference().DeliverAsync(envelope); + await receiver.AsReference().DeliverAsync(envelope); } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs index fe0f26549d0..8fffccd36a4 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxEnvelopeValidationTests.cs @@ -32,7 +32,7 @@ public async Task DefaultMessage_RejectsBeforeAcceptanceOrDuplicate(bool existin var exception = await Assert.ThrowsAsync(() => DeliverAsync(receiver, malformed)); - IInboxExtension extension = Fixture.GetInbox(receiver); + IMessagingExtension extension = Fixture.GetInbox(receiver); await OnTurnAsync(context, () => { var direct = extension.DeliverAsync(malformed, TestContext.Current.CancellationToken); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs index 8cdccb8240b..ddb7a2cdf37 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -75,7 +75,7 @@ public async Task PendingDuplicate_ChangedCustomHeadersPreservesOriginal() var schedules = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, context.GrainId); await OnTurnAsync(context, async () => { - IInboxExtension extension = context.ActivationServices.GetRequiredService(); + IMessagingExtension extension = context.ActivationServices.GetRequiredService(); Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(changed)).Status); var retained = Assert.Single(pending.Messages()).Envelope; Assert.Same(first.Payload, retained.Payload); @@ -189,7 +189,7 @@ await OnTurnAsync(context, () => var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId)); await OnTurnAsync(context, async () => { - IInboxExtension extension = context.ActivationServices.GetRequiredService(); + IMessagingExtension extension = context.ActivationServices.GetRequiredService(); await Assert.ThrowsAsync(() => extension.DeliverAsync(message, TestContext.Current.CancellationToken).AsTask()); }); Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(context.GrainId))); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs index 3e1aebd0140..6adb385a6c6 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -399,7 +399,7 @@ private static Task DeliverOnTurnAsync(Rig rig, InboxMessage env try { Assert.Same(rig.Context, ReceiverTestServices.CurrentGrainContext); - IInboxExtension extension = rig.Context.ActivationServices.GetRequiredService(); + IMessagingExtension extension = rig.Context.ActivationServices.GetRequiredService(); started.SetResult(extension.DeliverAsync(envelope, TestContext.Current.CancellationToken).AsTask()); } catch (Exception exception) { started.SetException(exception); } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 9bef196de4e..260ea4517ff 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -31,7 +31,7 @@ public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndTheSinkReportsLat { var receiver = NewGrain(); var message = NewInboxMessage("deduplicated", NewPayload()); - var extension = receiver.AsReference(); + var extension = receiver.AsReference(); Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); @@ -57,7 +57,7 @@ public async Task SinkRejection_DeadLettersTheMessageAndRecordsItAsProcessed(Sin Probe.For(receiver).Mode = mode; var payload = NewPayload(); var message = NewInboxMessage("rejected", payload); - var extension = receiver.AsReference(); + var extension = receiver.AsReference(); var result = await extension.DeliverAsync(message, Token); @@ -88,7 +88,7 @@ public async Task SinkBackpressure_StagesNothingAndTheRetryIsAccepted() Probe.For(receiver).Mode = SinkMode.Backpressure; var payload = NewPayload(); var message = NewInboxMessage("backpressured", payload); - var extension = receiver.AsReference(); + var extension = receiver.AsReference(); Assert.Equal(DeliveryStatus.Backpressured, (await extension.DeliverAsync(message, Token)).Status); var (count, letters) = await OnGrainAsync(receiver, context => ( @@ -109,7 +109,7 @@ public async Task SinkBackpressure_StagesNothingAndTheRetryIsAccepted() public async Task ScheduledMessage_IsOfferedToTheSinkBeforeItIsDue() { var receiver = NewGrain(); - var extension = receiver.AsReference(); + var extension = receiver.AsReference(); var payload = NewPayload(); var scheduled = NewInboxMessage("scheduled", payload, DateTimeOffset.UtcNow.AddDays(1)); @@ -126,7 +126,7 @@ public async Task ScheduledMessage_IsOfferedToTheSinkBeforeItIsDue() public async Task DeclinedMessages_UseInboxAcceptanceAndHandlers() { var receiver = NewGrain(); - var extension = receiver.AsReference(); + var extension = receiver.AsReference(); Probe.For(receiver).Mode = SinkMode.Decline; var scheduled = NewInboxMessage("scheduled", NewPayload(), DateTimeOffset.UtcNow.AddDays(1)); diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs index 8bf6e751ecb..e32182de271 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs @@ -19,7 +19,7 @@ public async Task PendingAndProcessedDuplicates_RecordOneBoundedReceiptEach() await receiver.GetSnapshotAsync(); var context = Fixture.GetGrainContext(receiver); var services = context.ActivationServices; - IInboxExtension extension = services.GetRequiredService(); + IMessagingExtension extension = services.GetRequiredService(); // The pump runs alongside grain calls, so holding the handler keeps the accepted message pending. using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/metric-pending"); var envelope = CreateEnvelope(receiver, NewMessage(451, "duplicate-receipts"), "messages/metric-pending"); diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs index 6ae92eba69d..36956ae4add 100644 --- a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -51,7 +51,7 @@ public void AddMessaging_RepeatedPublicRegistrationRetainsOneConfiguratorAndScop Assert.Equal(ServiceLifetime.Scoped, descriptor.Lifetime); } - // The configurator binds the inbox extension at activation; the only keyed grain extension is the durable-jobs receiver. + // The configurator binds the messaging extension at activation; the only keyed grain extension is the durable-jobs receiver. var extension = Assert.Single(services, descriptor => descriptor.ServiceType == typeof(IGrainExtension)); Assert.Equal(typeof(Orleans.DurableJobs.DurableJob).Assembly, Assert.IsAssignableFrom(extension.ServiceKey).Assembly); } diff --git a/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs index ba6a14e97e3..72d300ada57 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingBehaviorTestBase.cs @@ -46,7 +46,7 @@ protected static async Task DeliverWithCancellationAsync( IMessagingTestGrain receiver, InboxMessage envelope, CancellationToken cancellationToken) => - await receiver.AsReference().DeliverAsync(envelope, cancellationToken); + await receiver.AsReference().DeliverAsync(envelope, cancellationToken); protected static async Task WaitForBarrierAsync( IMessagingTestGrain receiver, From a4019b4a9b5d64d66c938ff613d9fa525ce63c5b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 17:58:01 -0700 Subject: [PATCH 45/95] feat(messaging)!: demote failed deliveries to later batches and retry backpressure indefinitely The test fixtures configure MaxHandlerNotFoundRetryAge instead of the removed MaxDeliveryAttempts. --- .../Functional/MessageSourceAndSinkTests.cs | 2 +- test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 260ea4517ff..e98a4d95dc0 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -241,7 +241,7 @@ public async ValueTask InitializeAsync() { options.DeduplicationWindow = DeduplicationWindow; options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); - options.MaxDeliveryAttempts = 2; + options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(100); options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(10); }); silo.Services.AddSingleton(Probe); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index cd97dd92ee6..9bc2d71c17b 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -233,7 +233,7 @@ protected virtual void ConfigureOptions(DurableMessagingOptions options) options.DeduplicationWindow = TimeSpan.FromMinutes(10); options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); options.MaxProcessingAttempts = 1; - options.MaxDeliveryAttempts = 3; + options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(200); options.MaxRetainedDeadLetters = 2; options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(25); From 99a24363614da41ed60468cb2f2d66eb9679da25 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 19:19:33 -0700 Subject: [PATCH 46/95] refactor(messaging)!: track one outbox retry period and drop MaxHandlerNotFoundRetryAge OutboxMessageState records a single RetryingSince, which starts with the first failed attempt and restarts on each Backpressured answer. MaxOutboxRetryAge (now 6 hours) bounds both unanswered attempts and HandlerNotFound answers, and dead letters keep the last failure as their reason. --- .../Functional/MessageSourceAndSinkTests.cs | 3 +-- .../Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index e98a4d95dc0..21db0a43a51 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -240,8 +240,7 @@ public async ValueTask InitializeAsync() silo.AddMessaging(options => { options.DeduplicationWindow = DeduplicationWindow; - options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); - options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(100); + options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3); options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(10); }); silo.Services.AddSingleton(Probe); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index 9bc2d71c17b..96e67dde95b 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -231,9 +231,8 @@ protected virtual void ConfigureOptions(DurableMessagingOptions options) { options.MaxCapacity = 2; options.DeduplicationWindow = TimeSpan.FromMinutes(10); - options.MaxOutboxRetryAge = TimeSpan.FromMinutes(5); + options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3); options.MaxProcessingAttempts = 1; - options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(200); options.MaxRetainedDeadLetters = 2; options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(25); From 2cfea8fa0d5de85ea0c651e9caafec37f13cce65 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 08:42:00 -0700 Subject: [PATCH 47/95] refactor(messaging)!: align outbox retry terminology with Temporal Rename the retry options and state after Temporal's retry policy: - DurableMessagingOptions.BackpressureRetryDelay becomes RetryInitialInterval. - DurableMessagingOptions.MaxOutboxRetryAge becomes OutboxRetryTimeout. - OutboxMessageState.RetryingSince becomes ExpirationTime and stores the deadline: the first failed attempt plus OutboxRetryTimeout. A Backpressured answer clears it, and the next failed attempt sets a new deadline. BREAKING CHANGE: BackpressureRetryDelay and MaxOutboxRetryAge are renamed to RetryInitialInterval and OutboxRetryTimeout. --- src/Orleans.Messaging/Inbox.cs | 2 +- .../Functional/InboxHandlerCompletionTests.cs | 2 +- .../Functional/InboxHeaderBehaviorTests.cs | 2 +- test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs | 2 +- .../Functional/MessageSourceAndSinkTests.cs | 4 ++-- .../Support/MessagingClusterFixture.cs | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 85d31eb3413..2d29420f4b3 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -321,7 +321,7 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell AttemptCount = attempts, LastError = error.ToString(), NextAttemptAt = MessagingTime.AddClamped(TimeProvider.GetUtcNow(), - TimeSpan.FromTicks(Options.BackpressureRetryDelay.Ticks * (1L << Math.Min(attempts - 1, DurableMessagingOptions.MaximumBackoffExponent)))) + TimeSpan.FromTicks(Options.RetryInitialInterval.Ticks * (1L << Math.Min(attempts - 1, DurableMessagingOptions.MaximumBackoffExponent)))) } }); status = "retry"; diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs index 70aef48bae0..df9ed7ebe19 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -160,7 +160,7 @@ protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); options.MaxProcessingAttempts = 5; - options.BackpressureRetryDelay = TimeSpan.FromHours(1); + options.RetryInitialInterval = TimeSpan.FromHours(1); options.DeduplicationWindow = TimeSpan.FromDays(7); } } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs index ddb7a2cdf37..3d5a951bd3d 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -254,7 +254,7 @@ protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); options.MaxProcessingAttempts = 2; - options.BackpressureRetryDelay = TimeSpan.FromHours(1); + options.RetryInitialInterval = TimeSpan.FromHours(1); } } } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs index 5d3616d37f0..fd21017fd1c 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -153,7 +153,7 @@ protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); options.MaxProcessingAttempts = 3; - options.BackpressureRetryDelay = RetryDelay; + options.RetryInitialInterval = RetryDelay; } } } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 21db0a43a51..3192f63f4aa 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -240,8 +240,8 @@ public async ValueTask InitializeAsync() silo.AddMessaging(options => { options.DeduplicationWindow = DeduplicationWindow; - options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3); - options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(10); + options.OutboxRetryTimeout = TimeSpan.FromSeconds(3); + options.RetryInitialInterval = TimeSpan.FromMilliseconds(10); }); silo.Services.AddSingleton(Probe); silo.Services.RemoveAll(); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index 96e67dde95b..f25a47e60bd 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -231,11 +231,11 @@ protected virtual void ConfigureOptions(DurableMessagingOptions options) { options.MaxCapacity = 2; options.DeduplicationWindow = TimeSpan.FromMinutes(10); - options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3); + options.OutboxRetryTimeout = TimeSpan.FromSeconds(3); options.MaxProcessingAttempts = 1; options.MaxRetainedDeadLetters = 2; options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); - options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(25); + options.RetryInitialInterval = TimeSpan.FromMilliseconds(25); options.InboxBatchSize = 8; options.OutboxBatchSize = 8; } @@ -266,6 +266,6 @@ protected override void ConfigureOptions(DurableMessagingOptions options) base.ConfigureOptions(options); options.MaxCapacity = 1; options.MaxProcessingAttempts = 2; - options.BackpressureRetryDelay = TimeSpan.FromHours(1); + options.RetryInitialInterval = TimeSpan.FromHours(1); } } From 70fa5c1fee5b27386b0bd67a5326f029b04e4a11 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 15:08:30 -0700 Subject: [PATCH 48/95] feat(messaging): keep activations alive while the pump has inbox work The messaging pump holds the activation while a pass services queued inbox work, so idle collection does not interrupt running handlers. Work which only becomes due later relies on the durable job wake-up, so the activation can be collected while it waits. --- src/Orleans.Messaging/MessagingPump.cs | 20 +++++++++++ .../Functional/MessagingKeepAliveTests.cs | 33 +++++++++++++++++++ .../Functional/ScheduledDeliveryTests.cs | 22 +++++++++++++ .../Support/MessagingClusterFixture.cs | 17 ++++++++++ 4 files changed, 92 insertions(+) create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingKeepAliveTests.cs diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index fa88c4be84e..3353d200740 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -15,6 +15,8 @@ namespace Orleans.Messaging; /// A single loop runs on the activation scheduler and services the inbox whenever it is signaled. /// One durable job keeps waking the grain while the inbox holds work. Wake-ups may be early, /// late, or duplicated: every pass derives the work to perform from current state. +/// The loop keeps the activation alive while a pass runs over queued work. +/// Work which only becomes due later relies on the durable job, so the activation can be collected while it waits. /// internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecycleObserver { @@ -27,6 +29,7 @@ internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecy private TaskCompletionSource? _nextPass; private Task? _scheduling; private bool _wakeupScheduled; + private bool _keepingAlive; // Advances whenever a job run stops guaranteeing a future wake-up. A schedule requested in an earlier epoch may // belong to a job which has already run and completed, so it no longer guarantees a wake-up. @@ -169,6 +172,7 @@ private async Task RunAsync() var cancellationToken = _shutdown.Token; while (!cancellationToken.IsCancellationRequested) { + SetKeepAlive(Inbox.Count > 0); var pass = _nextPass; _nextPass = null; var progress = false; @@ -190,13 +194,29 @@ private async Task RunAsync() pass?.TrySetResult(); if (!progress) { + SetKeepAlive(false); await _signal.WaitAsync().ConfigureAwait(true); } } + SetKeepAlive(false); _nextPass?.TrySetCanceled(cancellationToken); } + private void SetKeepAlive(bool keepAlive) + { + if (keepAlive == _keepingAlive || GrainContext is not IActivationKeepAlive activation) return; + _keepingAlive = keepAlive; + if (keepAlive) + { + activation.AcquireKeepAlive(); + } + else + { + activation.ReleaseKeepAlive(); + } + } + public async Task OnStart(CancellationToken cancellationToken) { await Inbox.StartAsync(cancellationToken).ConfigureAwait(true); diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingKeepAliveTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingKeepAliveTests.cs new file mode 100644 index 00000000000..8dc3674466c --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingKeepAliveTests.cs @@ -0,0 +1,33 @@ +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingKeepAliveTests : MessagingBehaviorTestBase +{ + private const string HeldRoute = "messages/held"; + + [Fact] + public async Task RunningHandler_KeepsActivationAliveUntilInboxDrains() + { + var receiver = NewGrain(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute); + var envelope = CreateEnvelope(receiver, NewMessage(1, "held"), HeldRoute); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var context = Fixture.GetGrainContext(receiver); + + // The handler runs outside of any grain request, so only the pump's hold prevents idle collection. + var management = Fixture.Client.GetGrain(0); + await management.ForceActivationCollection(TimeSpan.Zero, TestContext.Current.CancellationToken); + Assert.False(context.Deactivated.IsCompleted); + + handler.Release(); + Assert.Equal("held", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + await Fixture.CollectAsync(context); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs index e0e30fb2676..7c11678c3ff 100644 --- a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs @@ -168,6 +168,28 @@ public async Task ScheduledMessage_RedeliveryCoalescesAndDifferentDeliverAtConfl Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); } + [Fact] + public async Task ScheduledMessage_LeavesActivationCollectableUntilDue() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var envelope = Schedule(CreateEnvelope(receiver, NewMessage(50, "collected")), deliverAt); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await AssertWakeupAtAsync(receiver, deliverAt); + + // Work which is due later does not keep the activation alive; the durable wake-up reactivates it. + var context = Fixture.GetGrainContext(receiver); + await Fixture.CollectAsync(context); + + var reactivated = await receiver.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, reactivated.ActivationId); + Assert.Equal(1, reactivated.InboxCount); + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + Assert.Equal("collected", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + } + private async Task AssertWakeupAtAsync(IMessagingTestGrain grain, DateTimeOffset deliverAt) { var result = await Fixture.RunPumpAsync(grain); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index f25a47e60bd..44238fc719d 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -204,6 +204,23 @@ internal async Task WaitUntilAsync(IAddressable grain, Func predicate) } } + /// Forces idle activation collection until deactivates. + internal async Task CollectAsync(IGrainContext context) + { + var management = Client.GetGrain(0); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (!context.Deactivated.IsCompleted) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Activation '{context.GrainId}' was not collected."); + } + + await management.ForceActivationCollection(TimeSpan.Zero, TestContext.Current.CancellationToken); + await Task.Delay(50, TestContext.Current.CancellationToken); + } + } + internal IGrainContext GetGrainContext(IAddressable grain) { if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) From 2a9a30b323dfb5665bba768c66960619043b6bd1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 07:06:12 -0700 Subject: [PATCH 49/95] feat(messaging)!: bound inbox handler retries by time instead of attempts A handler exception leaves a message's outcome unknown. Retrying it a fixed number of times (MaxProcessingAttempts, about 15 seconds of backoff by default) dead-letters messages during outages and deployments far shorter than the outbox's six-hour delivery window. InboxRetryTimeout replaces MaxProcessingAttempts. A message whose handler attempts throw retries with exponential backoff until the timeout elapses after its first failure; the last attempt runs at the timeout. Fail(reason) remains the immediate, permanent path. InboxMessageState records the expiration so the bound survives reactivation. --- .../Configuration/DurableMessagingOptions.cs | 19 ++++-- src/Orleans.Messaging/Inbox.cs | 12 +++- src/Orleans.Messaging/MessageState.cs | 3 + src/Orleans.Messaging/README.md | 5 +- .../Orleans.Messaging/Orleans.Messaging.cs | 4 +- .../DeliveryAndOptionsContractTests.cs | 4 +- .../Functional/InboxHandlerCompletionTests.cs | 2 +- .../Functional/InboxHeaderBehaviorTests.cs | 2 +- .../Functional/InboxRetryTests.cs | 64 +++++++++++++++---- .../Support/MessagingClusterFixture.cs | 4 +- .../Support/MessagingTestGrains.cs | 8 ++- 11 files changed, 97 insertions(+), 30 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index 57aedc62e39..190fb977fab 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -103,9 +103,20 @@ public class DurableMessagingOptions public TimeSpan RetryInitialInterval { get; set; } = TimeSpan.FromSeconds(1); /// - /// Gets or sets the maximum number of attempts before an inbox message is dead-lettered. + /// Gets or sets how long the inbox retries a message whose handler attempts throw before the message is + /// dead-lettered. /// - public int MaxProcessingAttempts { get; set; } = 5; + /// + /// The timeout starts with the first failed attempt, and the last attempt runs when it elapses. A handler exception + /// leaves the message's outcome unknown, so retries span transient faults, such as outages and deployments which + /// fix a bug; a handler calls IInboxHandlerContext.Fail to dead-letter a message at once. A retrying message + /// keeps its place in . + /// + /// + /// Must be greater than or equal to , which dead-letters a message at its first failure. + /// Defaults to 6 hours. + /// + public TimeSpan InboxRetryTimeout { get; set; } = TimeSpan.FromHours(6); /// /// Gets or sets how long the outbox retries a message whose delivery attempts fail, such as during an @@ -201,9 +212,9 @@ public void Validate() $"RetryInitialInterval must be less than or equal to {MaxSupportedRetryDelay}."); } - if (MaxProcessingAttempts <= 0) + if (InboxRetryTimeout < TimeSpan.Zero) { - throw new ArgumentOutOfRangeException(nameof(MaxProcessingAttempts), MaxProcessingAttempts, "MaxProcessingAttempts must be greater than zero."); + throw new ArgumentOutOfRangeException(nameof(InboxRetryTimeout), InboxRetryTimeout, "InboxRetryTimeout must be greater than or equal to TimeSpan.Zero."); } if (OutboxRetryTimeout <= TimeSpan.Zero) diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 2d29420f4b3..bc9b1b2b386 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -304,14 +304,20 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell return; } + // An exception leaves the outcome unknown, so the message retries until InboxRetryTimeout after its first + // failure. The last attempt runs when the timeout elapses; Fail(reason) dead-letters a message at once. var attempts = checked(entry.State.AttemptCount + 1); - if (attempts >= Options.MaxProcessingAttempts) + var now = TimeProvider.GetUtcNow(); + var expirationTime = entry.State.ExpirationTime ?? MessagingTime.AddClamped(now, Options.InboxRetryTimeout); + if (expirationTime <= now) { StageDeadLetter(entry.Message, error.Message, attempts); status = "dead_lettered"; } else { + var retryAt = MessagingTime.AddClamped(now, + TimeSpan.FromTicks(Options.RetryInitialInterval.Ticks * (1L << Math.Min(attempts - 1, DurableMessagingOptions.MaximumBackoffExponent)))); Emit(new() { Kind = InboxCommandKind.Retry, @@ -320,8 +326,8 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell { AttemptCount = attempts, LastError = error.ToString(), - NextAttemptAt = MessagingTime.AddClamped(TimeProvider.GetUtcNow(), - TimeSpan.FromTicks(Options.RetryInitialInterval.Ticks * (1L << Math.Min(attempts - 1, DurableMessagingOptions.MaximumBackoffExponent)))) + NextAttemptAt = retryAt < expirationTime ? retryAt : expirationTime, + ExpirationTime = expirationTime } }); status = "retry"; diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs index 0fa6e5401ad..e8a18f4d25a 100644 --- a/src/Orleans.Messaging/MessageState.cs +++ b/src/Orleans.Messaging/MessageState.cs @@ -14,6 +14,9 @@ internal sealed class InboxMessageState [Id(2)] public string? LastError { get; set; } + /// Gets or sets when a failing message's retries end, InboxRetryTimeout after its first failure. + [Id(3)] + public DateTimeOffset? ExpirationTime { get; set; } } [GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 06b514a0d95..1780e230744 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -67,8 +67,9 @@ A handler finishes each message with `IInboxHandlerContext.Complete()` or - `Complete()` stages the message's removal and completion record. - `Fail(reason)` moves the message to the inbox dead letters and records its ID as processed. -- A handler exception before either call retries the message with exponential - backoff, and dead-letters it after `MaxProcessingAttempts`. +- A handler exception before either call leaves the outcome unknown, so the message + retries with exponential backoff until `InboxRetryTimeout` (6 hours) after its + first failure, then moves to the dead letters. - Returning without either call, or calling both, deactivates the grain. The inbox persists each batch's outcomes and business changes in one journal write. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 7e389e2c18e..50234269c41 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -355,9 +355,9 @@ public partial class DurableMessagingOptions public int InboxBatchSize { get { throw null; } set { } } - public int MaxCapacity { get { throw null; } set { } } + public System.TimeSpan InboxRetryTimeout { get { throw null; } set { } } - public int MaxProcessingAttempts { get { throw null; } set { } } + public int MaxCapacity { get { throw null; } set { } } public int MaxRetainedDeadLetters { get { throw null; } set { } } diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index 48d6c014d12..530f7c8e593 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -82,7 +82,7 @@ public void Validate_DefaultOptions_SucceedsAndExposesDocumentedDefaults() Assert.Equal(TimeSpan.FromDays(7), options.DeduplicationWindow); Assert.Equal(TimeSpan.FromHours(6), options.OutboxRetryTimeout); Assert.Equal(TimeSpan.FromDays(30), options.DeadLetterRetentionPeriod); - Assert.Equal(5, options.MaxProcessingAttempts); + Assert.Equal(TimeSpan.FromHours(6), options.InboxRetryTimeout); Assert.Same(ResiliencePipeline.Empty, options.DeliveryResiliencePipeline); Assert.Equal(1000, options.MaxRetainedDeadLetters); Assert.Equal(32, options.InboxBatchSize); @@ -98,7 +98,7 @@ public void Validate_EachCapacityRetryDeadLetterAndBatchBoundary_EnforcesContrac (nameof(DurableMessagingOptions.DeduplicationWindow), options => options.DeduplicationWindow = TimeSpan.Zero), (nameof(DurableMessagingOptions.RetryInitialInterval), options => options.RetryInitialInterval = TimeSpan.Zero), (nameof(DurableMessagingOptions.RetryInitialInterval), options => options.RetryInitialInterval = TimeSpan.MaxValue), - (nameof(DurableMessagingOptions.MaxProcessingAttempts), options => options.MaxProcessingAttempts = 0), + (nameof(DurableMessagingOptions.InboxRetryTimeout), options => options.InboxRetryTimeout = TimeSpan.FromTicks(-1)), (nameof(DurableMessagingOptions.OutboxRetryTimeout), options => options.OutboxRetryTimeout = TimeSpan.Zero), (nameof(DurableMessagingOptions.DeadLetterRetentionPeriod), options => options.DeadLetterRetentionPeriod = TimeSpan.Zero), (nameof(DurableMessagingOptions.MaxRetainedDeadLetters), options => options.MaxRetainedDeadLetters = 0), diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs index df9ed7ebe19..873e50d90c5 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -159,7 +159,7 @@ private sealed class PermanentFailureClusterFixture : MessagingClusterFixture protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); - options.MaxProcessingAttempts = 5; + options.InboxRetryTimeout = TimeSpan.FromHours(4); options.RetryInitialInterval = TimeSpan.FromHours(1); options.DeduplicationWindow = TimeSpan.FromDays(7); } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs index 3d5a951bd3d..ec29753275d 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -253,7 +253,7 @@ private sealed class HeaderFixture : MessagingClusterFixture protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); - options.MaxProcessingAttempts = 2; + options.InboxRetryTimeout = TimeSpan.FromHours(1); options.RetryInitialInterval = TimeSpan.FromHours(1); } } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs index fd21017fd1c..53fd21101c0 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -13,6 +13,9 @@ public sealed class InboxRetryTests() : MessagingBehaviorTestBase(new RetryFixtu { private static readonly TimeSpan RetryDelay = TimeSpan.FromMinutes(1); + // Attempts run at 0, 1, 3, 7, 15, 31 and 63 retry delays; the eighth and last runs when the timeout elapses. + private static readonly TimeSpan RetryTimeout = RetryDelay * 100; + [Fact] public async Task HandlerFailure_RetriesAfterBackoffAndEarlyWakeupsAreHarmless() { @@ -51,31 +54,70 @@ public async Task HandlerFailure_RetriesAfterBackoffAndEarlyWakeupsAreHarmless() } [Fact] - public async Task OrdinaryHandlerFailures_DeadLetterAtExactConfiguredAttemptLimit() + public async Task TransientHandlerFailures_RetryUntilTheHandlerSucceedsWithinTheRetryTimeout() { var receiver = NewGrain(); _ = await receiver.GetSnapshotAsync(); var inbox = Fixture.GetInbox(receiver); - var envelope = CreateEnvelope(receiver, NewMessage(193, "always-fails") with { ThrowDuringPreparation = true }); + var envelope = CreateEnvelope(receiver, NewMessage(196, "transient") with { ThrowForAttempts = 6 }); var messageId = envelope.Envelope.MessageId; var start = Fixture.Clock.GetUtcNow(); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); + await RunFailedAttemptsAsync(receiver, inbox, messageId, start, failedAttempts: 6); - // Backoff doubles per attempt. - Fixture.Clock.Advance(RetryDelay); - _ = await Fixture.RunPumpAsync(receiver); - await Fixture.WaitUntilAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 2); - var second = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId)); - Assert.Equal(start + RetryDelay + RetryDelay * 2, second.NextAttemptAt); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(1, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + // Verifies each failed attempt's retry state, running the pump at each retry, and leaves the clock at the attempt + // which follows the last failed one. + private async Task RunFailedAttemptsAsync(IMessagingTestGrain receiver, Inbox inbox, MessageId messageId, DateTimeOffset start, int failedAttempts) + { + var expirationTime = start + RetryTimeout; + for (var attempt = 1; ; attempt++) + { + var state = await Fixture.OnGrainAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId)); + Assert.Equal(attempt, state.AttemptCount); + Assert.Equal(expirationTime, state.ExpirationTime); + var backoff = start + RetryDelay * ((1 << attempt) - 1); + Assert.Equal(backoff < expirationTime ? backoff : expirationTime, state.NextAttemptAt); + Fixture.Clock.Advance(state.NextAttemptAt!.Value - Fixture.Clock.GetUtcNow()); + if (attempt == failedAttempts) + { + return; + } + + _ = await Fixture.RunPumpAsync(receiver); + await Fixture.WaitUntilAsync(receiver, () => InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == attempt + 1); + } + } + [Fact] + public async Task OrdinaryHandlerFailures_RetryUntilTheRetryTimeoutThenDeadLetter() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var inbox = Fixture.GetInbox(receiver); + var envelope = CreateEnvelope(receiver, NewMessage(193, "always-fails") with { ThrowDuringPreparation = true }); + var messageId = envelope.Envelope.MessageId; + var start = Fixture.Clock.GetUtcNow(); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await Fixture.WaitUntilAsync(receiver, () => inbox.Count == 1 && InboxTestAccess.GetMessageState(inbox, messageId).AttemptCount == 1); - Fixture.Clock.Advance(RetryDelay * 2); + // Backoff doubles per attempt, and the last attempt runs when InboxRetryTimeout elapses after the first failure. + await RunFailedAttemptsAsync(receiver, inbox, messageId, start, failedAttempts: 7); + Assert.Equal(start + RetryTimeout, Fixture.Clock.GetUtcNow()); Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); var deadLettered = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); var deadLetter = Assert.Single(deadLettered.InboxDeadLetters); Assert.Equal(messageId, deadLetter.MessageId); - Assert.Equal(3, deadLetter.AttemptCount); + Assert.Equal(8, deadLetter.AttemptCount); Assert.Equal(0, deadLettered.InboxCount); Assert.Empty(deadLettered.Effects); Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); @@ -152,7 +194,7 @@ private sealed class RetryFixture : MessagingClusterFixture protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); - options.MaxProcessingAttempts = 3; + options.InboxRetryTimeout = RetryTimeout; options.RetryInitialInterval = RetryDelay; } } diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index 44238fc719d..8b6363342b2 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -249,7 +249,7 @@ protected virtual void ConfigureOptions(DurableMessagingOptions options) options.MaxCapacity = 2; options.DeduplicationWindow = TimeSpan.FromMinutes(10); options.OutboxRetryTimeout = TimeSpan.FromSeconds(3); - options.MaxProcessingAttempts = 1; + options.InboxRetryTimeout = TimeSpan.Zero; options.MaxRetainedDeadLetters = 2; options.DeadLetterRetentionPeriod = TimeSpan.FromHours(1); options.RetryInitialInterval = TimeSpan.FromMilliseconds(25); @@ -282,7 +282,7 @@ protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); options.MaxCapacity = 1; - options.MaxProcessingAttempts = 2; + options.InboxRetryTimeout = TimeSpan.FromHours(1); options.RetryInitialInterval = TimeSpan.FromHours(1); } } diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index fd2223133c4..36ffdbc1eb9 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -40,7 +40,8 @@ public sealed record TestMessage( [property: Id(8)] bool ThrowDuringPreparation = false, [property: Id(5)] bool CommitDuringHandling = false, [property: Id(6)] bool DeleteDuringHandling = false, - [property: Id(9)] bool ThrowOnceDuringPreparation = false); + [property: Id(9)] bool ThrowOnceDuringPreparation = false, + [property: Id(10)] int ThrowForAttempts = 0); [GenerateSerializer, Immutable] public sealed record MessageEffect( @@ -98,6 +99,7 @@ internal Exception? JournalPreparationFailure private int _maxConcurrentHandlers; // Spans activations so a replayed message does not fail again after recovery. private static readonly ConcurrentDictionary FailedOnce = new(); + private static readonly ConcurrentDictionary FailedAttempts = new(); public MessagingTestGrain( IGrainContext grainContext, @@ -371,7 +373,9 @@ private async ValueTask HandleAsync( { await WriteStateAsync(cancellationToken); } - if (message.ThrowDuringPreparation || (message.ThrowOnceDuringPreparation && FailedOnce.TryAdd(message.LogicalId, 0))) + if (message.ThrowDuringPreparation + || (message.ThrowOnceDuringPreparation && FailedOnce.TryAdd(message.LogicalId, 0)) + || (message.ThrowForAttempts > 0 && FailedAttempts.AddOrUpdate(message.LogicalId, 1, static (_, count) => count + 1) <= message.ThrowForAttempts)) { throw new InvalidOperationException($"Injected handler preparation failure for {message.LogicalId}."); } From dc7c2531a86868b6577f6e091559403b11485531 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 07:41:05 -0700 Subject: [PATCH 50/95] feat(messaging): wake grains for work held by registered inbox work sources A component that accepts work through an inbox sink holds it in its own journaled state, so the inbox and outbox are empty and the pump scheduled no wake-up: a grain that deactivated or crashed with such work runnable was never reactivated. IInbox.Register(IInboxWorkSource) lets that component report pending work and the longest acceptable wake-up delay. Every journal write that leaves work pending ensures the pump's durable job before it is stored, and the job reschedules itself by the shortest source delay until no source reports work. --- src/Orleans.Messaging/IInbox.cs | 9 + src/Orleans.Messaging/IInboxWorkSource.cs | 34 ++++ src/Orleans.Messaging/Inbox.State.cs | 6 +- src/Orleans.Messaging/Inbox.cs | 31 ++- src/Orleans.Messaging/MessagingPump.cs | 10 +- src/Orleans.Messaging/README.md | 11 +- .../Orleans.Messaging/Orleans.Messaging.cs | 6 + .../Functional/InboxWorkSourceTests.cs | 188 ++++++++++++++++++ .../Support/TestInbox.cs | 2 + 9 files changed, 285 insertions(+), 12 deletions(-) create mode 100644 src/Orleans.Messaging/IInboxWorkSource.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs diff --git a/src/Orleans.Messaging/IInbox.cs b/src/Orleans.Messaging/IInbox.cs index cc5142e16d4..db6cb3a8655 100644 --- a/src/Orleans.Messaging/IInbox.cs +++ b/src/Orleans.Messaging/IInbox.cs @@ -29,4 +29,13 @@ public interface IInbox /// Register sinks during construction or activation setup, before the grain receives messages. /// IInbox Register(IInboxMessageSink sink); + + /// Registers a source of durable work which the inbox's wake-up keeps reactivating the grain for. + /// The work source. + /// This inbox. + /// + /// See . Register work sources during construction or activation setup, before the + /// grain activates. + /// + IInbox Register(IInboxWorkSource source); } diff --git a/src/Orleans.Messaging/IInboxWorkSource.cs b/src/Orleans.Messaging/IInboxWorkSource.cs new file mode 100644 index 00000000000..7a3a367a9b8 --- /dev/null +++ b/src/Orleans.Messaging/IInboxWorkSource.cs @@ -0,0 +1,34 @@ +using System; + +namespace Orleans.Messaging; + +/// +/// Reports durable work which a component holds outside the inbox, such as messages which an +/// accepted, so the inbox's durable wake-up reactivates the grain while that work remains. +/// +/// +/// +/// The inbox keeps one durable wake-up job per grain while it, the outbox, or any registered work source holds work. +/// Before each journal write, the inbox consults the work sources and ensures the wake-up, so a write which makes work +/// pending also guarantees that the grain is reactivated after a deactivation or crash. Hold the work in journaled +/// state of the grain's journal, and resume it when the grain activates: a wake-up reactivates the grain, and the work +/// source's owner continues its work from the recovered state. +/// +/// +/// Each wake-up job run consults the work sources again. It reschedules after the shortest delay any source reports, +/// and the wake-up ends once no source, the inbox, nor the outbox holds work. Wake-ups may be early, late, or duplicated, +/// including while the grain is active and running the work. +/// +/// +public interface IInboxWorkSource +{ + /// Gets the delay before the grain must be woken for this source's pending work. + /// + /// when the source holds no pending work; otherwise, the delay after which a wake-up should + /// reactivate the grain if it isn't active. A delay of or less requests a prompt + /// wake-up. For work which runs as soon as the grain is active, return the longest delay acceptable before a + /// deactivated grain resumes it. + /// + /// Runs synchronously on the activation scheduler, before each journal write and on each wake-up. + TimeSpan? GetWakeupDelay(); +} diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index 8c570960704..5b647647efd 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -25,8 +25,12 @@ internal sealed partial class Inbox private bool IsDurable(InboxEntry entry) => entry.Sequence <= _durableSequence; + // Every journal write prepares every state, so a write which makes a work source's work pending ensures the wake-up + // before it can become durable. ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) => - _lastSequence != _capturedSequence ? _pump.EnsureWakeupAsync(cancellationToken) : ValueTask.CompletedTask; + _lastSequence != _capturedSequence || GetWorkSourceDelay() is not null + ? _pump.EnsureWakeupAsync(cancellationToken) + : ValueTask.CompletedTask; private void OnWriteCaptured() => _capturedSequence = _lastSequence; diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index bc9b1b2b386..170919c72f2 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -26,6 +26,7 @@ internal sealed partial class Inbox : IInbox, IMessagingExtension, IStateMachine private readonly MessagingPump _pump; private readonly InboxDispatcher _handlers = new([]); private readonly List _sinks = []; + private readonly List _workSources = []; public Inbox(MessagingPump pump) { @@ -61,6 +62,13 @@ public IInbox Register(IInboxMessageSink sink) return this; } + public IInbox Register(IInboxWorkSource source) + { + ArgumentNullException.ThrowIfNull(source); + _workSources.Add(source); + return this; + } + public async ValueTask DeliverAsync(InboxMessage message, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -394,20 +402,35 @@ internal bool RemoveDeadLetter(MessageId messageId) return true; } + /// Gets a value indicating whether the inbox or a registered work source holds pending work. + internal bool HasWork => _messages.Count > 0 || GetWorkSourceDelay() is not null; + /// - /// Returns the delay until the next message is due, or when the inbox is empty. + /// Returns the delay until the next message or work source is due, or when neither the inbox + /// nor any work source holds work. /// /// The current Durable Jobs time, which the caller uses to schedule the wake-up. internal TimeSpan? GetNextWakeupDelay(DateTimeOffset jobNow) { - if (_messages.Count == 0) return null; + var next = GetWorkSourceDelay(); + if (_messages.Count == 0 || next <= TimeSpan.Zero) return next; var now = TimeProvider.GetUtcNow(); - var next = TimeSpan.MaxValue; foreach (var entry in _messages.Values) { var delay = GetDelay(entry, now, jobNow); if (delay == TimeSpan.Zero) return delay; - if (delay < next) next = delay; + if (next is null || delay < next) next = delay; + } + + return next; + } + + private TimeSpan? GetWorkSourceDelay() + { + TimeSpan? next = null; + foreach (var source in _workSources) + { + if (source.GetWakeupDelay() is { } delay && (next is null || delay < next)) next = delay; } return next; diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 3353d200740..3d589772e05 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -13,8 +13,8 @@ namespace Orleans.Messaging; /// /// /// A single loop runs on the activation scheduler and services the inbox whenever it is signaled. -/// One durable job keeps waking the grain while the inbox holds work. Wake-ups may be early, -/// late, or duplicated: every pass derives the work to perform from current state. +/// One durable job keeps waking the grain while the inbox or a registered holds work. +/// Wake-ups may be early, late, or duplicated: every pass derives the work to perform from current state. /// The loop keeps the activation alive while a pass runs over queued work. /// Work which only becomes due later relies on the durable job, so the activation can be collected while it waits. /// @@ -113,7 +113,7 @@ public async ValueTask EnsureWakeupAsync(CancellationToken cancellationToken) public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) { - // A running job keeps rescheduling itself until the inbox is empty. + // A running job keeps rescheduling itself until the inbox and every work source are empty. _wakeupScheduled = true; EnsureStarted(); var pass = _nextPass ??= new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -126,7 +126,7 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte { // This attempt no longer guarantees a future wake-up. ClearWakeup(); - if (!_shutdown.IsCancellationRequested && Inbox.Count > 0) + if (!_shutdown.IsCancellationRequested && Inbox.HasWork) { RepairWakeupAsync().Ignore(); } @@ -220,7 +220,7 @@ private void SetKeepAlive(bool keepAlive) public async Task OnStart(CancellationToken cancellationToken) { await Inbox.StartAsync(cancellationToken).ConfigureAwait(true); - if (Inbox.Count > 0) + if (Inbox.HasWork) { await EnsureWakeupAsync(cancellationToken).ConfigureAwait(true); } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 1780e230744..70817d13e18 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -121,6 +121,12 @@ construction or activation setup, before journal recovery. sink honors their `DeliverAt`. Declined messages follow the handler path once `DeliverAt` passes. A message is offered again after its processed record leaves `DeduplicationWindow`, so sinks must be idempotent. +- `IInbox.Register(IInboxWorkSource)` adds a work source: a component, such as a + sink, that holds durable work in the grain's journal. `GetWakeupDelay` returns how + soon a deactivated grain must wake to resume that work, or `null` when none is + pending. Every journal write that leaves work pending schedules the messaging + wake-up before it is stored, and the wake-up reactivates the grain until the work + clears. The source's owner resumes its work on activation. ## Delivery and processing @@ -136,8 +142,9 @@ receiver returns a `DeliveryResult`: - `HandlerNotFound`: the receiver has no registered handlers. Each activation runs one messaging pump on its scheduler, processing inbox batches. -One durable job per grain wakes the pump while the inbox holds work. Journaled state determines which messages are due, so each -wake-up processes exactly the ready work. +One durable job per grain wakes the pump while the inbox or a registered work source +holds work. Journaled state determines which messages are due, so each wake-up +processes exactly the ready work. ## State diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 50234269c41..9f4795d5a6c 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -175,6 +175,7 @@ public partial interface IInbox { IInbox Register(IInboxHandler handler); IInbox Register(IInboxMessageSink sink); + IInbox Register(IInboxWorkSource source); } public partial interface IInboxHandler @@ -196,6 +197,11 @@ public partial interface IInboxMessageSink bool TryAccept(InboxMessage message, out DeliveryResult result); } + public partial interface IInboxWorkSource + { + System.TimeSpan? GetWakeupDelay(); + } + public partial interface IMessagingDiagnostics { System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs new file mode 100644 index 00000000000..f58c8354e4e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs @@ -0,0 +1,188 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// The inbox's durable wake-up reactivates a grain while a registered holds work. +/// +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class InboxWorkSourceTests() : MessagingBehaviorTestBase(receiverOnly: false) +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + [Fact] + public async Task WriteWhichMakesWorkPending_SchedulesTheWakeupBeforeTheWriteIsStored() + { + var grain = NewWorkGrain(); + await grain.GetPendingAsync(); + using var write = Fixture.Storage.BlockWrite(Journal(grain)); + + var adding = grain.AddWorkAsync(TimeSpan.Zero); + await write.WaitUntilEnteredAsync(); + + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(MessagingPump.JobName, grain.GetGrainId())); + write.Release(); + await adding; + await grain.CompleteWorkAsync(); + } + + [Fact] + public async Task CrashAfterTheWriteWhichMakesWorkPending_ReactivatesTheGrain() + { + var grain = NewWorkGrain(); + await grain.GetPendingAsync(); + var crashed = Fixture.GetGrainContext(grain); + + // The write becomes durable but its acknowledgement fails, so the activation fences and stops. + Fixture.Storage.FailAfterWrite(Journal(grain)); + await Assert.ThrowsAsync(() => grain.AddWorkAsync(TimeSpan.Zero)); + await crashed.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + await WaitForActivationsAsync(grain, 2); + Assert.Equal(1, await grain.GetPendingAsync()); + await grain.CompleteWorkAsync(); + } + + [Fact] + public async Task DeactivationWithPendingWork_ReactivatesTheGrain() + { + var grain = NewWorkGrain(); + await grain.AddWorkAsync(TimeSpan.Zero); + var previous = Fixture.GetGrainContext(grain); + + await grain.DeactivateAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + await WaitForActivationsAsync(grain, 2); + Assert.Equal(1, await grain.GetPendingAsync()); + await grain.CompleteWorkAsync(); + } + + [Fact] + public async Task Wakeup_ReschedulesAfterTheSourceDelay_AndCompletesOnceTheWorkClears() + { + var delay = TimeSpan.FromHours(1); + var grain = NewWorkGrain(); + await grain.AddWorkAsync(delay); + + var before = DateTimeOffset.UtcNow; + var pending = await Fixture.RunPumpAsync(grain); + var after = DateTimeOffset.UtcNow; + Assert.Equal(DurableJobRunStatus.RescheduleRequested, pending.Status); + Assert.InRange(pending.RescheduleTime!.Value, before + delay, after + delay); + + await grain.CompleteWorkAsync(); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + } + + [Fact] + public async Task WritesWithoutPendingWork_DoNotScheduleAWakeup() + { + var grain = NewWorkGrain(); + + await grain.AddWorkAsync(TimeSpan.Zero); + await grain.CompleteWorkAsync(); + var scheduled = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grain.GetGrainId()); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + + await grain.TouchAsync(); + Assert.Equal(scheduled, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grain.GetGrainId())); + } + + private IWorkSourceTestGrain NewWorkGrain() => Fixture.Client.GetGrain(Guid.NewGuid()); + + private static JournalId Journal(IAddressable grain) => JournalId.FromGrainId(grain.GetGrainId()); + + private static async Task WaitForActivationsAsync(IAddressable grain, int expected) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (WorkSourceTestGrain.GetActivationCount(grain.GetGrainId()) < expected) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Grain '{grain.GetGrainId()}' was not reactivated."); + } + + await Task.Delay(10, Token); + } + } +} + +public interface IWorkSourceTestGrain : IGrainWithGuidKey +{ + Task AddWorkAsync(TimeSpan wakeupDelay); + Task CompleteWorkAsync(); + Task TouchAsync(); + Task GetPendingAsync(); + Task DeactivateAsync(); +} + +/// Holds pending work in journaled state and reports it as an inbox work source. +public sealed class WorkSourceTestGrain : Grain, IWorkSourceTestGrain, IMessagingGrain, IInboxWorkSource +{ + private static readonly ConcurrentDictionary Activations = new(); + private readonly IJournaledStateManager _manager; + private readonly IDurableValue _pending; + private readonly IDurableValue _wakeupDelayTicks; + private readonly IDurableValue _touches; + + public WorkSourceTestGrain( + IJournaledStateManager manager, + IInbox inbox, + [FromKeyedServices("work-pending")] IDurableValue pending, + [FromKeyedServices("work-wakeup-delay")] IDurableValue wakeupDelayTicks, + [FromKeyedServices("work-touches")] IDurableValue touches) + { + _manager = manager; + _pending = pending; + _wakeupDelayTicks = wakeupDelayTicks; + _touches = touches; + inbox.Register(this); + } + + internal static int GetActivationCount(GrainId grainId) => Activations.TryGetValue(grainId, out var count) ? count : 0; + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Activations.AddOrUpdate(this.GetGrainId(), 1, static (_, count) => count + 1); + return Task.CompletedTask; + } + + public async Task AddWorkAsync(TimeSpan wakeupDelay) + { + _pending.Value++; + _wakeupDelayTicks.Value = wakeupDelay.Ticks; + await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task CompleteWorkAsync() + { + _pending.Value = 0; + await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task TouchAsync() + { + _touches.Value++; + await _manager.WriteStateAsync(CancellationToken.None); + } + + public Task GetPendingAsync() => Task.FromResult(_pending.Value); + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } + + TimeSpan? IInboxWorkSource.GetWakeupDelay() => + _pending.Value > 0 ? TimeSpan.FromTicks(_wakeupDelayTicks.Value) : null; +} diff --git a/test/Orleans.Messaging.Tests/Support/TestInbox.cs b/test/Orleans.Messaging.Tests/Support/TestInbox.cs index 2429da08869..2714042b8e0 100644 --- a/test/Orleans.Messaging.Tests/Support/TestInbox.cs +++ b/test/Orleans.Messaging.Tests/Support/TestInbox.cs @@ -15,6 +15,8 @@ public IInbox Register(IInboxHandler handler) public IInbox Register(IInboxMessageSink sink) => throw new NotSupportedException("The test inbox dispatches to handlers only."); + public IInbox Register(IInboxWorkSource source) => throw new NotSupportedException("The test inbox dispatches to handlers only."); + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) => _dispatcher.HandleAsync(context, cancellationToken); } From 3fb3b97225e709da07fb18c3b649618b831396eb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:29:21 -0700 Subject: [PATCH 51/95] fix(messaging): acknowledge duplicates only after their completion is durable A duplicate of a completed message returned Duplicate as soon as the completion was staged, before the write recording it was durable. Track the sequence of the last staged completion and persist before acknowledging a duplicate while that completion is not yet durable. --- src/Orleans.Messaging/Inbox.State.cs | 14 ++++++++---- src/Orleans.Messaging/Inbox.cs | 6 ++--- .../InboxAcceptanceBehaviorTests.cs | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index 5b647647efd..ff4f4c1fb60 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -18,17 +18,22 @@ internal sealed partial class Inbox // Pending messages with a delivery time, which do not count toward MaxCapacity. private int _scheduledCount; - // Entries are durable once a completed write captured their sequence number. + // Acceptances and completions are durable once a completed write captured their sequence number. private long _lastSequence; private long _capturedSequence; private long _durableSequence; + private long _lastAcceptSequence; + private long _lastCompletionSequence; private bool IsDurable(InboxEntry entry) => entry.Sequence <= _durableSequence; + // Whether every processed record which deduplication relies on is durable. + private bool AreCompletionsDurable => _lastCompletionSequence <= _durableSequence; + // Every journal write prepares every state, so a write which makes a work source's work pending ensures the wake-up // before it can become durable. ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) => - _lastSequence != _capturedSequence || GetWorkSourceDelay() is not null + _lastAcceptSequence > _capturedSequence || GetWorkSourceDelay() is not null ? _pump.EnsureWakeupAsync(cancellationToken) : ValueTask.CompletedTask; @@ -54,7 +59,7 @@ void IStateMachine.Reset(JournalStreamWriter writer) _processed.Clear(); _deadLetters.Clear(); _scheduledCount = 0; - _lastSequence = _capturedSequence = _durableSequence = 0; + _lastSequence = _capturedSequence = _durableSequence = _lastAcceptSequence = _lastCompletionSequence = 0; } void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => @@ -115,7 +120,7 @@ private void Apply(InboxCommand command) { case InboxCommandKind.Accept: var message = command.Message ?? throw new InvalidOperationException("An accept command requires its message."); - _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = ++_lastSequence }); + _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = _lastAcceptSequence = ++_lastSequence }); _processed.Remove(message.Envelope.MessageId); if (message.Envelope.DeliverAt is not null) _scheduledCount++; ReportDepth(1); @@ -131,6 +136,7 @@ private void Apply(InboxCommand command) } _processed[command.MessageId] = command.ProcessedAt; + _lastCompletionSequence = ++_lastSequence; if (command.DeadLetter is { } letter) _deadLetters[command.MessageId] = letter; break; case InboxCommandKind.RemoveDeadLetters: diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 170919c72f2..684a24ec38f 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -82,9 +82,9 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel { if (IsDuplicate(message, out var pending)) { - // The sender may retire its copy once this returns, so the acceptance must be durable. A sink acceptance - // journals only its processed record, which can still be in flight. - if (pending is null ? _sinks.Count > 0 : !IsDurable(pending)) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); + // The sender may retire its copy once this returns, so the acceptance or the processed record must be + // durable. A completion staged by a handler or a sink can still be in flight. + if (pending is null ? !AreCompletionsDurable : !IsDurable(pending)) await PersistForCallerAsync(cancellationToken).ConfigureAwait(true); if (pending is not null) _pump.Signal(); Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs index 5c1b4b3cdc0..aa0eede5e04 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxAcceptanceBehaviorTests.cs @@ -74,6 +74,28 @@ public async Task ConcurrentDuplicateDeliveries_ConvergeToOneEffectWithinRetenti Assert.Equal(1, state.MaxConcurrentHandlers); } + [Fact] + public async Task DuplicateOfCompletedMessage_WaitsUntilCompletionIsDurable() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var journalId = JournalId.FromGrainId(receiver.GetGrainId()); + var envelope = CreateEnvelope(receiver, NewMessage(12, "completion-durability")); + + // The first write accepts the message and the second records its completion. + var barrier = Fixture.Storage.BlockWrite(journalId, matchingWrite: 2); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await barrier.WaitUntilEnteredAsync(); + + var duplicate = DeliverAsync(receiver, envelope); + await Task.WhenAny(duplicate, Task.Delay(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken)); + Assert.False(duplicate.IsCompleted); + + barrier.Release(); + Assert.Equal(DeliveryStatus.Duplicate, (await duplicate).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + } + [Fact] public async Task DuplicateAfterReactivationWithinRetention_RemainsEffectivelyOnce() { From 6db875e49e67c67c9df1806f3ce3d14af583515c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:30:22 -0700 Subject: [PATCH 52/95] fix(messaging): count sink dead-letters as received dead_lettered A message which an inbox sink dead-letters was counted as received 'accepted', making it indistinguishable from a sink acceptance. --- src/Orleans.Messaging/Inbox.cs | 2 +- src/Orleans.Messaging/README.md | 4 +++- .../Functional/MessageSourceAndSinkTests.cs | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 684a24ec38f..3d4a7b1ca57 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -184,7 +184,7 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel throw; } - Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, accepted ? "accepted" : "dead_lettered"); Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, accepted ? "success" : "dead_lettered"); if (!accepted) return DeliveryResult.DeadLettered(reason); LogMessageAccepted(Logger, key, GrainContext.GrainId); diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 70817d13e18..0e59ee35f7e 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -164,4 +164,6 @@ durably at the grain's next journal write. The `Microsoft.Orleans` meter publishes counters for received and processed messages tagged by grain type and status, a histogram for processing duration, and the -`orleans-messaging-inbox-depth` up-down counter for pending work. +`orleans-messaging-inbox-depth` up-down counter for pending work. A message an inbox +sink accepts counts as received `accepted` and processed `success`; one which the +sink dead-letters counts as received and processed `dead_lettered`. diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 3192f63f4aa..0d61686eb58 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -58,11 +58,16 @@ public async Task SinkRejection_DeadLettersTheMessageAndRecordsItAsProcessed(Sin var payload = NewPayload(); var message = NewInboxMessage("rejected", payload); var extension = receiver.AsReference(); + await receiver.WriteAsync(); + using var metrics = new MessageMetricListener(await OnGrainAsync(receiver, context => context.ActivationServices.GetRequiredService().Meter)); var result = await extension.DeliverAsync(message, Token); Assert.Equal(DeliveryStatus.DeadLettered, result.Status); Assert.Equal(reason, result.Message); + var grainType = receiver.GetGrainId().Type.ToString(); + Assert.Single(metrics.Read("inbox-messages-received")).AssertCounter(grainType, "dead_lettered"); + Assert.Single(metrics.Read("inbox-messages-processed")).AssertCounter(grainType, "dead_lettered"); var (state, count, letters) = await OnGrainAsync(receiver, context => ( Grain(context).GetState(), Inbox(context).Count, From 573dc2946677db6446cca9044ceed936de030487 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:30:57 -0700 Subject: [PATCH 53/95] fix(messaging): resume pending messages when a handler registers instead of polling A pending message without a registered handler kept the pump's durable job rescheduling itself every second. Messages now count toward the wake-up only when a handler is registered, and registering a handler while messages are pending signals the pump and ensures a wake-up. --- src/Orleans.Messaging/IInboxHandler.cs | 1 + src/Orleans.Messaging/Inbox.cs | 8 ++++++-- src/Orleans.Messaging/MessagingPump.cs | 11 ++++++++++- .../Functional/InboxRetryTests.cs | 11 +++++------ .../Support/MessagingTestGrains.cs | 10 +++++++++- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/Orleans.Messaging/IInboxHandler.cs b/src/Orleans.Messaging/IInboxHandler.cs index 9d5d707a9ac..90ff6f89589 100644 --- a/src/Orleans.Messaging/IInboxHandler.cs +++ b/src/Orleans.Messaging/IInboxHandler.cs @@ -22,3 +22,4 @@ public interface IInboxHandler /// A task which completes when handling returns. ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken); } + diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 3d4a7b1ca57..fc843932485 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -52,6 +52,9 @@ public Inbox(MessagingPump pump) public IInbox Register(IInboxHandler handler) { _handlers.Add(handler); + + // Pending messages wait for a handler without holding a wake-up, so a late registration resumes them. + if (_messages.Count > 0) _pump.RequestWakeup(); return this; } @@ -407,13 +410,14 @@ internal bool RemoveDeadLetter(MessageId messageId) /// /// Returns the delay until the next message or work source is due, or when neither the inbox - /// nor any work source holds work. + /// nor any work source holds work which this activation can perform. Messages wait without a wake-up until a handler + /// is registered. /// /// The current Durable Jobs time, which the caller uses to schedule the wake-up. internal TimeSpan? GetNextWakeupDelay(DateTimeOffset jobNow) { var next = GetWorkSourceDelay(); - if (_messages.Count == 0 || next <= TimeSpan.Zero) return next; + if (_messages.Count == 0 || _handlers.IsEmpty || next <= TimeSpan.Zero) return next; var now = TimeProvider.GetUtcNow(); foreach (var entry in _messages.Values) { diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 3d589772e05..432d6d294f5 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -13,7 +13,8 @@ namespace Orleans.Messaging; /// /// /// A single loop runs on the activation scheduler and services the inbox whenever it is signaled. -/// One durable job keeps waking the grain while the inbox or a registered holds work. +/// One durable job keeps waking the grain while the inbox or a registered holds work +/// which the activation can perform; inbox messages wait for a handler without a wake-up. /// Wake-ups may be early, late, or duplicated: every pass derives the work to perform from current state. /// The loop keeps the activation alive while a pass runs over queued work. /// Work which only becomes due later relies on the durable job, so the activation can be collected while it waits. @@ -74,6 +75,14 @@ public void Signal() if (!_shutdown.IsCancellationRequested) _signal.Signal(); } + /// Requests a processing pass and ensures a durable wake-up for work which has become processable. + public void RequestWakeup() + { + if (_shutdown.IsCancellationRequested) return; + _signal.Signal(); + RepairWakeupAsync().Ignore(); + } + /// Starts the loop once the activation is servicing grain calls. public void EnsureStarted() { diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs index 53fd21101c0..ecf99623913 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -150,16 +150,15 @@ public async Task AcceptedMessage_WaitsForHandlerRegistrationWithoutDeadLetterin Assert.Equal(1, withoutHandler.InboxCount); Assert.Empty(withoutHandler.Effects); Assert.Empty(withoutHandler.InboxDeadLetters); - Assert.NotEqual(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); + + // Without a handler, the pending message holds no wake-up. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); Assert.Equal(1, (await receiver.GetSnapshotAsync()).InboxCount); + // Registering a handler on the same activation resumes the pending message. await receiver.ConfigureHandlerAsync(true); - var noHandlerContext = Fixture.GetGrainContext(receiver); - await receiver.RequestDeactivationAsync(); - await noHandlerContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); - _ = await receiver.GetSnapshotAsync(); - _ = await Fixture.RunPumpAsync(receiver); var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal(withoutHandler.ActivationId, completed.ActivationId); Assert.Equal("awaits-handler", Assert.Single(completed.Effects).Value); Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); } diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 36ffdbc1eb9..eeb7fb0a24b 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -97,6 +97,7 @@ internal Exception? JournalPreparationFailure private readonly Guid _activationId = Guid.NewGuid(); private int _activeHandlers; private int _maxConcurrentHandlers; + private bool _handlerRegistered; // Spans activations so a replayed message does not fail again after recovery. private static readonly ConcurrentDictionary FailedOnce = new(); private static readonly ConcurrentDictionary FailedAttempts = new(); @@ -140,7 +141,7 @@ public override async Task OnActivateAsync(CancellationToken cancellationToken) { if (_handlerConfiguration.IsEnabled(this.GetGrainId())) { - _inbox.Register(new ApplicationDispatcher(this)); + RegisterHandler(); } await base.OnActivateAsync(cancellationToken); _snapshotProbe.Publish(this.GetGrainId(), CreateSnapshot()); @@ -203,9 +204,16 @@ public Task StageEffectAsync(MessageEffect effect) public Task ConfigureHandlerAsync(bool enabled) { _handlerConfiguration.Set(this.GetGrainId(), enabled); + if (enabled && !_handlerRegistered) RegisterHandler(); return Task.CompletedTask; } + private void RegisterHandler() + { + _inbox.Register(new ApplicationDispatcher(this)); + _handlerRegistered = true; + } + public async Task RemoveInboxDeadLetterAsync(MessageId messageId) { if (!_diagnostics.RemoveInboxDeadLetter(messageId)) From ca1a218f17e0c7bf25d11c3f9db298ca049940b5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:31:07 -0700 Subject: [PATCH 54/95] fix(messaging): schedule inbox retries on the Durable Jobs clock Inbox retry backoff and InboxRetryTimeout deadlines now use the Durable Jobs clock, the same time domain as DeliverAt and the wake-ups which run each retry. A retry's due time and its wake-up therefore agree when the grain clock and the Durable Jobs clock differ. --- .../Configuration/DurableMessagingOptions.cs | 3 +- src/Orleans.Messaging/Inbox.cs | 25 +++++++++-------- src/Orleans.Messaging/MessageState.cs | 6 +++- .../Functional/InboxRetryTests.cs | 7 +++++ .../Support/MessagingClusterFixture.cs | 28 +++++++++++++++++++ 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index 190fb977fab..df88f501b39 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -110,7 +110,8 @@ public class DurableMessagingOptions /// The timeout starts with the first failed attempt, and the last attempt runs when it elapses. A handler exception /// leaves the message's outcome unknown, so retries span transient faults, such as outages and deployments which /// fix a bug; a handler calls IInboxHandlerContext.Fail to dead-letter a message at once. A retrying message - /// keeps its place in . + /// keeps its place in . The timeout and retry backoff follow the Durable Jobs clock, which + /// schedules the wake-up for each retry. /// /// /// Must be greater than or equal to , which dead-letters a message at its first failure. diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index fc843932485..c257b91d6be 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -237,12 +237,11 @@ internal async ValueTask ProcessReadyAsync(CancellationToken cancellationT return false; } - var now = TimeProvider.GetUtcNow(); var jobNow = Shared.JobTimeProvider.GetUtcNow(); var batch = new List(Math.Min(Options.InboxBatchSize, _messages.Count)); foreach (var entry in _messages.Values) { - if (IsDurable(entry) && GetDelay(entry, now, jobNow) == TimeSpan.Zero) + if (IsDurable(entry) && GetDelay(entry, jobNow) == TimeSpan.Zero) { batch.Add(entry); if (batch.Count == Options.InboxBatchSize) break; @@ -317,8 +316,9 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell // An exception leaves the outcome unknown, so the message retries until InboxRetryTimeout after its first // failure. The last attempt runs when the timeout elapses; Fail(reason) dead-letters a message at once. + // Retry deadlines follow the Durable Jobs clock, which schedules the wake-up for the next attempt. var attempts = checked(entry.State.AttemptCount + 1); - var now = TimeProvider.GetUtcNow(); + var now = Shared.JobTimeProvider.GetUtcNow(); var expirationTime = entry.State.ExpirationTime ?? MessagingTime.AddClamped(now, Options.InboxRetryTimeout); if (expirationTime <= now) { @@ -418,10 +418,9 @@ internal bool RemoveDeadLetter(MessageId messageId) { var next = GetWorkSourceDelay(); if (_messages.Count == 0 || _handlers.IsEmpty || next <= TimeSpan.Zero) return next; - var now = TimeProvider.GetUtcNow(); foreach (var entry in _messages.Values) { - var delay = GetDelay(entry, now, jobNow); + var delay = GetDelay(entry, jobNow); if (delay == TimeSpan.Zero) return delay; if (next is null || delay < next) next = delay; } @@ -440,13 +439,17 @@ internal bool RemoveDeadLetter(MessageId messageId) return next; } - // DeliverAt is in the Durable Jobs time domain, which schedules wake-ups; retry backoff follows the grain clock. - private static TimeSpan GetDelay(InboxEntry entry, DateTimeOffset now, DateTimeOffset jobNow) + // DeliverAt and retry deadlines are in the Durable Jobs time domain, which schedules wake-ups. + private static TimeSpan GetDelay(InboxEntry entry, DateTimeOffset jobNow) + { + var dueTime = GetDueTime(entry); + return dueTime > jobNow ? dueTime - jobNow : TimeSpan.Zero; + } + + private static DateTimeOffset GetDueTime(InboxEntry entry) { - var delay = TimeSpan.Zero; - if (entry.Message.Envelope.DeliverAt is { } deliverAt && deliverAt > jobNow) delay = deliverAt - jobNow; - if (entry.State.NextAttemptAt is { } retryAt && retryAt > now && retryAt - now > delay) delay = retryAt - now; - return delay; + var dueTime = entry.Message.Envelope.DeliverAt ?? DateTimeOffset.MinValue; + return entry.State.NextAttemptAt is { } retryAt && retryAt > dueTime ? retryAt : dueTime; } internal async Task StartAsync(CancellationToken cancellationToken) diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs index e8a18f4d25a..4294365ce1a 100644 --- a/src/Orleans.Messaging/MessageState.cs +++ b/src/Orleans.Messaging/MessageState.cs @@ -8,13 +8,17 @@ internal sealed class InboxMessageState [Id(0)] public int AttemptCount { get; set; } + /// Gets or sets the time, on the Durable Jobs clock, from which the inbox attempts the message again. [Id(1)] public DateTimeOffset? NextAttemptAt { get; set; } [Id(2)] public string? LastError { get; set; } - /// Gets or sets when a failing message's retries end, InboxRetryTimeout after its first failure. + /// + /// Gets or sets when a failing message's retries end on the Durable Jobs clock, InboxRetryTimeout after its + /// first failure. + /// [Id(3)] public DateTimeOffset? ExpirationTime { get; set; } } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs index ecf99623913..b2409b6d3e7 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.DependencyInjection; using Orleans.DurableJobs; using Orleans.Messaging.Configuration; using Orleans.Messaging.Tests.Support; @@ -196,5 +197,11 @@ protected override void ConfigureOptions(DurableMessagingOptions options) options.InboxRetryTimeout = RetryTimeout; options.RetryInitialInterval = RetryDelay; } + + protected override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + UseTestClockForDurableJobs(services); + } } } diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index 8b6363342b2..b3a6d5eefe1 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -49,6 +49,10 @@ protected MessagingClusterFixture(int initialSilos, bool receiverOnly = false, b }); siloBuilder.Services.AddSingleton(Clock); siloBuilder.Services.UseTimeProviderForBackgroundAreas(TimeProvider.System); + // Advancing the test clock also advances the Durable Jobs clock, which inbox retry deadlines follow. + siloBuilder.Services.AddKeyedSingleton( + DurableJobTimeProviderNames.DurableJobs, + new FollowingJobClock(Clock, Clock.GetUtcNow())); siloBuilder.Services.AddSingleton(HandlerProbe); siloBuilder.Services.AddSingleton(new TestHandlerConfiguration()); siloBuilder.Services.AddSingleton(SnapshotProbe); @@ -85,6 +89,24 @@ protected MessagingClusterFixture(int initialSilos, bool receiverOnly = false, b Cluster = builder.Build(); } + /// + /// Makes the Durable Jobs clock read , so retry deadlines are exact test-clock times. Durable Jobs + /// timers keep following real time and run each wake-up once the test clock reaches its due time. + /// + protected void UseTestClockForDurableJobs(IServiceCollection services) => + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, new TestJobClock(Clock)); + + /// A Durable Jobs clock which follows real time plus the test clock's advance since the fixture started. + private sealed class FollowingJobClock(TimeProvider clock, DateTimeOffset clockStart) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => System.GetUtcNow() + (clock.GetUtcNow() - clockStart); + } + + private sealed class TestJobClock(TimeProvider clock) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); + } + public InProcessTestCluster Cluster { get; } public IClusterClient Client => Cluster.Client!; public FakeTimeProvider Clock { get; } @@ -285,4 +307,10 @@ protected override void ConfigureOptions(DurableMessagingOptions options) options.InboxRetryTimeout = TimeSpan.FromHours(1); options.RetryInitialInterval = TimeSpan.FromHours(1); } + + protected override void ConfigureServices(IServiceCollection services) + { + base.ConfigureServices(services); + UseTestClockForDurableJobs(services); + } } From 250fb359e6bcb1678a7dd2e465754282bdc01aa5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:32:19 -0700 Subject: [PATCH 55/95] fix(messaging): schedule an earlier wake-up for work due before the current one The pump tracks the job which wakes the grain next and its due time. Work staged with an earlier due time, including a DeliverAt time or retry, schedules an earlier wake-up; the job which a newer, earlier wake-up supersedes completes when it runs. Staged due times are cleared when a write captures them, so repeated and retried preparations re-ensure the wake-up. ManualJobClock timers fire at least every 100 ms so Durable Jobs starts shards and runs wake-ups promptly after a test advances the clock. --- src/Orleans.Messaging/Inbox.State.cs | 41 ++++++++--- src/Orleans.Messaging/MessagingPump.cs | 69 ++++++++++++++----- .../Functional/InboxRetryTests.cs | 2 +- .../Functional/ScheduledDeliveryTests.cs | 32 +++++++++ .../Support/InboxTestAccess.cs | 3 + .../Support/ManualJobClock.cs | 20 +++++- .../Support/MessagingClusterFixture.cs | 37 ++++++---- 7 files changed, 159 insertions(+), 45 deletions(-) diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index ff4f4c1fb60..c7c419a0aff 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -22,22 +22,35 @@ internal sealed partial class Inbox private long _lastSequence; private long _capturedSequence; private long _durableSequence; - private long _lastAcceptSequence; private long _lastCompletionSequence; + // The earliest due time, on the Durable Jobs clock, of accepted or retrying messages staged since the last capture. + private DateTimeOffset? _stagedWakeup; + private bool IsDurable(InboxEntry entry) => entry.Sequence <= _durableSequence; // Whether every processed record which deduplication relies on is durable. private bool AreCompletionsDurable => _lastCompletionSequence <= _durableSequence; - // Every journal write prepares every state, so a write which makes a work source's work pending ensures the wake-up - // before it can become durable. - ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) => - _lastAcceptSequence > _capturedSequence || GetWorkSourceDelay() is not null - ? _pump.EnsureWakeupAsync(cancellationToken) - : ValueTask.CompletedTask; + // Every journal write prepares every state, so a write which makes a message or a work source's work pending ensures + // its wake-up before it can become durable. + ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) + { + var dueTime = _stagedWakeup; + if (GetWorkSourceDelay() is { } delay) + { + var workDueTime = MessagingTime.AddClamped(Shared.JobTimeProvider.GetUtcNow(), delay); + if (dueTime is null || workDueTime < dueTime) dueTime = workDueTime; + } - private void OnWriteCaptured() => _capturedSequence = _lastSequence; + return dueTime is { } due ? _pump.EnsureWakeupAsync(due, cancellationToken) : ValueTask.CompletedTask; + } + + private void OnWriteCaptured() + { + _capturedSequence = _lastSequence; + _stagedWakeup = null; + } void IStateMachine.OnWriteCompleted() { @@ -49,6 +62,13 @@ private void Emit(InboxCommand command) { Shared.InboxCodec.WriteSet(command, _writer); Apply(command); + var dueTime = command switch + { + { Kind: InboxCommandKind.Accept, Message: { } message } => message.Envelope.DeliverAt ?? DateTimeOffset.MinValue, + { Kind: InboxCommandKind.Retry, State: { } state } => state.NextAttemptAt, + _ => (DateTimeOffset?)null + }; + if (dueTime is { } due && (_stagedWakeup is null || due < _stagedWakeup)) _stagedWakeup = due; } void IStateMachine.Reset(JournalStreamWriter writer) @@ -59,7 +79,8 @@ void IStateMachine.Reset(JournalStreamWriter writer) _processed.Clear(); _deadLetters.Clear(); _scheduledCount = 0; - _lastSequence = _capturedSequence = _durableSequence = _lastAcceptSequence = _lastCompletionSequence = 0; + _stagedWakeup = null; + _lastSequence = _capturedSequence = _durableSequence = _lastCompletionSequence = 0; } void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => @@ -120,7 +141,7 @@ private void Apply(InboxCommand command) { case InboxCommandKind.Accept: var message = command.Message ?? throw new InvalidOperationException("An accept command requires its message."); - _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = _lastAcceptSequence = ++_lastSequence }); + _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = ++_lastSequence }); _processed.Remove(message.Envelope.MessageId); if (message.Envelope.DeliverAt is not null) _scheduledCount++; ReportDepth(1); diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 432d6d294f5..c09d8adbffb 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -15,7 +15,10 @@ namespace Orleans.Messaging; /// A single loop runs on the activation scheduler and services the inbox whenever it is signaled. /// One durable job keeps waking the grain while the inbox or a registered holds work /// which the activation can perform; inbox messages wait for a handler without a wake-up. +/// Work staged with an earlier due time than the current wake-up schedules an earlier one, and the job which a newer, +/// earlier wake-up supersedes completes when it runs. /// Wake-ups may be early, late, or duplicated: every pass derives the work to perform from current state. +/// Wake-up bookkeeping runs on the activation scheduler: journal writes prepare on it and serialize with job runs. /// The loop keeps the activation alive while a pass runs over queued work. /// Work which only becomes due later relies on the durable job, so the activation can be collected while it waits. /// @@ -29,11 +32,15 @@ internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecy private Task? _loop; private TaskCompletionSource? _nextPass; private Task? _scheduling; - private bool _wakeupScheduled; + private DateTimeOffset _schedulingDueTime; private bool _keepingAlive; - // Advances whenever a job run stops guaranteeing a future wake-up. A schedule requested in an earlier epoch may - // belong to a job which has already run and completed, so it no longer guarantees a wake-up. + // The job which wakes the grain next, if any, and its due time. A job run which completes clears it. + private string? _wakeupJobId; + private DateTimeOffset _wakeupDueTime; + + // Advances whenever a job run completes. A schedule requested in an earlier epoch may belong to a job which has + // already run and completed, so it no longer guarantees a wake-up. private int _wakeupEpoch; private int _schedulingEpoch; @@ -92,18 +99,24 @@ public void EnsureStarted() } } - /// Ensures a durable job will wake this grain while messaging work remains. - public async ValueTask EnsureWakeupAsync(CancellationToken cancellationToken) + /// + /// Ensures a durable job will wake this grain no later than , on the Durable Jobs clock. + /// + public async ValueTask EnsureWakeupAsync(DateTimeOffset dueTime, CancellationToken cancellationToken) { - if (_wakeupScheduled) return; - if (_scheduling is null || _schedulingEpoch != _wakeupEpoch) + // Work which is already due needs a wake-up due now, which any tracked or scheduling wake-up due by now satisfies. + var now = Shared.JobTimeProvider.GetUtcNow(); + if (dueTime < now) dueTime = now; + if (_wakeupJobId is not null && _wakeupDueTime <= dueTime) return; + if (_scheduling is null || _schedulingEpoch != _wakeupEpoch || _schedulingDueTime > dueTime) { _schedulingEpoch = _wakeupEpoch; + _schedulingDueTime = dueTime; _scheduling = Shared.JobManager.ScheduleJobAsync(new ScheduleJobRequest { Target = GrainContext.GrainId, JobName = JobName, - DueTime = Shared.JobTimeProvider.GetUtcNow() + DueTime = _schedulingDueTime }, _shutdown.Token); } @@ -111,8 +124,12 @@ public async ValueTask EnsureWakeupAsync(CancellationToken cancellationToken) var epoch = _schedulingEpoch; try { - await scheduling.WaitAsync(cancellationToken).ConfigureAwait(true); - if (epoch == _wakeupEpoch) _wakeupScheduled = true; + var job = await scheduling.WaitAsync(cancellationToken).ConfigureAwait(true); + if (epoch == _wakeupEpoch && (_wakeupJobId is null || job.DueTime < _wakeupDueTime)) + { + _wakeupJobId = job.Id; + _wakeupDueTime = job.DueTime; + } } finally { @@ -123,7 +140,12 @@ public async ValueTask EnsureWakeupAsync(CancellationToken cancellationToken) public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) { // A running job keeps rescheduling itself until the inbox and every work source are empty. - _wakeupScheduled = true; + var jobId = context.Job.Id; + if (_wakeupJobId is null) + { + _wakeupJobId = jobId; + _wakeupDueTime = context.Job.DueTime; + } EnsureStarted(); var pass = _nextPass ??= new(TaskCreationOptions.RunContinuationsAsynchronously); _signal.Signal(); @@ -134,7 +156,7 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte catch { // This attempt no longer guarantees a future wake-up. - ClearWakeup(); + CompleteWakeup(jobId); if (!_shutdown.IsCancellationRequested && Inbox.HasWork) { RepairWakeupAsync().Ignore(); @@ -146,17 +168,26 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte var now = Shared.JobTimeProvider.GetUtcNow(); if (Inbox.GetNextWakeupDelay(now) is not { } delay) { - ClearWakeup(); + CompleteWakeup(jobId); + return DurableJobRunResult.Completed; + } + + var next = MessagingTime.AddClamped(now, delay > TimeSpan.Zero ? delay : RecheckDelay); + if (_wakeupJobId is not null && _wakeupJobId != jobId && _wakeupDueTime <= next) + { + // Another job wakes the grain first, and its run reschedules for any remaining work. + CompleteWakeup(jobId); return DurableJobRunResult.Completed; } - return DurableJobRunResult.RescheduleAt( - MessagingTime.AddClamped(now, delay > TimeSpan.Zero ? delay : RecheckDelay)); + _wakeupJobId = jobId; + _wakeupDueTime = next; + return DurableJobRunResult.RescheduleAt(next); } - private void ClearWakeup() + private void CompleteWakeup(string jobId) { - _wakeupScheduled = false; + if (_wakeupJobId == jobId) _wakeupJobId = null; _wakeupEpoch++; } @@ -164,7 +195,7 @@ private async Task RepairWakeupAsync() { try { - await EnsureWakeupAsync(CancellationToken.None).ConfigureAwait(true); + await EnsureWakeupAsync(Shared.JobTimeProvider.GetUtcNow(), CancellationToken.None).ConfigureAwait(true); } catch (Exception exception) when (!_shutdown.IsCancellationRequested) { @@ -231,7 +262,7 @@ public async Task OnStart(CancellationToken cancellationToken) await Inbox.StartAsync(cancellationToken).ConfigureAwait(true); if (Inbox.HasWork) { - await EnsureWakeupAsync(cancellationToken).ConfigureAwait(true); + await EnsureWakeupAsync(Shared.JobTimeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(true); } } diff --git a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs index b2409b6d3e7..7a4685da584 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxRetryTests.cs @@ -175,7 +175,7 @@ public async Task WakeupWhoseJobCompletesBeforeSchedulingReturns_DoesNotSuppress // The scheduled job runs and completes, with no work remaining, before the scheduling call returns. Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); using var schedule = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); - var scheduling = await Fixture.OnGrainAsync(receiver, () => pump.EnsureWakeupAsync(TestContext.Current.CancellationToken).AsTask()); + var scheduling = await Fixture.OnGrainAsync(receiver, () => pump.EnsureWakeupAsync(DateTimeOffset.MinValue, TestContext.Current.CancellationToken).AsTask()); await schedule.WaitUntilEnteredAsync(); Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(receiver)).Status); schedule.Continue(); diff --git a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs index 7c11678c3ff..60bcc531828 100644 --- a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs @@ -53,6 +53,38 @@ public async Task ScheduledMessage_IsHeldUntilDeliverAtThenProcessedOnce() Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); } + [Fact] + public async Task EarlierScheduledMessage_SchedulesAnEarlierWakeupWhichSupersedesTheLaterOne() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var grainId = receiver.GetGrainId(); + var later = JobClock.GetUtcNow() + Delay; + var earlier = JobClock.GetUtcNow() + TimeSpan.FromMinutes(1); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(20, "later")), later))).Status); + var laterJob = Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId)); + Assert.Equal(later, laterJob.DueTime); + + // The later wake-up does not cover a message due first, so its acceptance schedules an earlier one. + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(21, "earlier")), earlier))).Status); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId); + Assert.Equal(2, jobs.Count); + Assert.Equal(earlier, jobs[1].DueTime); + + // Work due later than the earliest wake-up schedules nothing further. + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, NewMessage(22, "last")), later + Delay))).Status); + Assert.Equal(2, Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grainId).Count); + + // Any other wake-up completes in favour of the earliest one, whose run reschedules for the remaining work. + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunDuplicatePumpAsync(receiver)).Status); + await AssertWakeupAtAsync(receiver, earlier); + JobClock.Advance(earlier - JobClock.GetUtcNow()); + await AssertWakeupAtAsync(receiver, later); + Assert.Equal("earlier", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunDuplicatePumpAsync(receiver)).Status); + } + [Theory] [InlineData(false)] [InlineData(true)] diff --git a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs index eb75bbe6768..0bf0554444b 100644 --- a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs +++ b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs @@ -36,6 +36,9 @@ public static void SetProcessedAt(Inbox inbox, MessageId messageId, DateTimeOffs public static InboxMessageState GetMessageState(Inbox inbox, MessageId messageId) => inbox.MessageState[messageId].State; + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_wakeupJobId")] + public static extern ref string? WakeupJobId(MessagingPump pump); + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_messages")] private static extern ref Dictionary GetMessages(Inbox inbox); diff --git a/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs b/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs index a3331d2fdc0..ae123397e30 100644 --- a/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs +++ b/test/Orleans.Messaging.Tests/Support/ManualJobClock.cs @@ -4,14 +4,30 @@ namespace Orleans.Messaging.Tests.Support; /// A Durable Jobs clock whose current time moves only when a test advances it. /// /// -/// Timers and timestamps keep following real time, so Durable Jobs keeps running wake-ups that are due, -/// while tests decide exactly when scheduled messages become due. +/// Timers follow real time and fire at least every , so Durable Jobs re-reads the clock, +/// starts shards, and runs wake-ups promptly once a test advances it, while tests decide exactly when work becomes due. /// public sealed class ManualJobClock(DateTimeOffset start) : TimeProvider { + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(100); private long _utcTicks = start.UtcTicks; + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + new PollingTimer(base.CreateTimer(callback, state, Clamp(dueTime), Clamp(period))); + public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _utcTicks), TimeSpan.Zero); public void Advance(TimeSpan delta) => Interlocked.Add(ref _utcTicks, delta.Ticks); + + private static TimeSpan Clamp(TimeSpan dueTime) => dueTime > PollInterval ? PollInterval : dueTime; + + // Task.Delay creates its timer disarmed and arms it through Change, so both paths clamp. + private sealed class PollingTimer(ITimer timer) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => timer.Change(Clamp(dueTime), Clamp(period)); + + public void Dispose() => timer.Dispose(); + + public ValueTask DisposeAsync() => timer.DisposeAsync(); + } } diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index b3a6d5eefe1..a083266c568 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -168,21 +168,32 @@ await services.GetRequiredService().ScheduleJobAsync( TestContext.Current.CancellationToken); } - /// Executes the pump's durable job callback directly, as a spurious or duplicate wake-up would. - internal async Task RunPumpAsync(IAddressable grain) + /// + /// Executes the pump's durable job callback directly as its current wake-up job, as an early or spurious firing of that + /// job would. + /// + internal Task RunPumpAsync(IAddressable grain) => + RunPumpAsync(grain, TestContext.Current.CancellationToken, duplicate: false); + + /// Executes the pump's durable job callback directly as a job other than its current wake-up job. + internal Task RunDuplicatePumpAsync(IAddressable grain) => + RunPumpAsync(grain, TestContext.Current.CancellationToken, duplicate: true); + + private async Task RunPumpAsync(IAddressable grain, CancellationToken runCancellation, bool duplicate) { - var job = new DurableJob - { - Id = Guid.NewGuid().ToString("N"), - Name = MessagingPump.JobName, - ShardId = "test", - TargetGrainId = grain.GetGrainId(), - DueTime = DateTimeOffset.UtcNow - }; var pump = GetPump(grain); - var run = await OnGrainAsync(grain, () => pump.ExecuteJobAsync( - new PumpRunContext(job), - TestContext.Current.CancellationToken).AsTask()); + var run = await OnGrainAsync(grain, () => + { + var job = new DurableJob + { + Id = (duplicate ? null : InboxTestAccess.WakeupJobId(pump)) ?? Guid.NewGuid().ToString("N"), + Name = MessagingPump.JobName, + ShardId = "test", + TargetGrainId = grain.GetGrainId(), + DueTime = DateTimeOffset.UtcNow + }; + return pump.ExecuteJobAsync(new PumpRunContext(job), runCancellation).AsTask(); + }); return await run.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); } From 176a484100ea812f4406757876d0c0ed3524311e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:32:38 -0700 Subject: [PATCH 56/95] feat(messaging)!: make inbox handler selection read-only IInboxHandler.CanHandle now receives the InboxMessage instead of the mutable IInboxHandlerContext. A handler can only stage a terminal outcome from HandleAsync, after the inbox selects it. BREAKING CHANGE: IInboxHandler.CanHandle(IInboxHandlerContext) is replaced by CanHandle(InboxMessage). Orleans.Messaging is alpha and has no package validation baseline. --- .../Functional/ByteMessagingPayloadTests.cs | 2 +- .../Functional/InboxHandlerCompletionTests.cs | 4 ++-- .../Functional/InboxHeaderBehaviorTests.cs | 2 +- test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs | 2 +- .../Functional/MessageSourceAndSinkTests.cs | 2 +- test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs | 2 +- .../Support/NamedFactoryMessagingGrain.cs | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs index a1a5b3fc994..2b50fd3307a 100644 --- a/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ByteMessagingPayloadTests.cs @@ -81,7 +81,7 @@ private sealed class PayloadProbe(byte[] expected) : IInboxHandler, IDisposable public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs index 873e50d90c5..9b4b0920cbf 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHandlerCompletionTests.cs @@ -140,7 +140,7 @@ private sealed class PermanentHandler : IInboxHandler { public int Attempts { get; private set; } - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) @@ -692,7 +692,7 @@ private sealed class TestHandler(IDurableDictionary ef public OutboxMessage Output { get; private set; } public Func Body { get; set; } = static (self, _) => { self.Mutate(); self.Context.Complete(); return default; }; - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { diff --git a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs index ec29753275d..0322274e8f7 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxHeaderBehaviorTests.cs @@ -231,7 +231,7 @@ private sealed class RawHandler(IGrainContext context) : IInboxHandler, IDisposa public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public async ValueTask HandleAsync(IInboxHandlerContext handler, CancellationToken cancellationToken) diff --git a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs index 6adb385a6c6..725647974ae 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxIdentityTests.cs @@ -440,7 +440,7 @@ private sealed class IdentityHandler : IInboxHandler, IDisposable public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public int Calls { get; private set; } - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 0d61686eb58..23a2a3a1e7a 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -403,7 +403,7 @@ bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result private sealed class Handler(SourceSinkProbe.Counters probe) : IInboxHandler { - public bool CanHandle(IInboxHandlerContext context) => context.Message.Envelope.Subject == Subject; + public bool CanHandle(InboxMessage message) => message.Envelope.Subject == Subject; public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index eeb7fb0a24b..75fbe07338f 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -441,7 +441,7 @@ private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => private sealed class ApplicationDispatcher(MessagingTestGrain owner) : IInboxHandler { - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { diff --git a/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs index d54c3bb08c8..6f20494c9b2 100644 --- a/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs +++ b/test/Orleans.Messaging.Tests/Support/NamedFactoryMessagingGrain.cs @@ -83,7 +83,7 @@ public Task RequestDeactivationAsync() } - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; From d15e5541af8fbcec4aee2eab3768597dd7f6ad3a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:32:48 -0700 Subject: [PATCH 57/95] fix(messaging): report null delivery pipeline through options validation DurableMessagingOptions.Validate throws ArgumentNullException for a null DeliveryResiliencePipeline. The registered validator now treats every ArgumentException as a validation failure, so the error surfaces as OptionsValidationException. --- .../Hosting/MessagingExtensions.cs | 2 +- .../Hosting/PublicMessagingRegistrationTests.cs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs index 8c7db451f7b..2122ffe224c 100644 --- a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -65,7 +65,7 @@ public static IServiceCollection AddMessaging(this IServiceCollection services, options.Validate(); return true; } - catch (ArgumentOutOfRangeException) + catch (ArgumentException) { return false; } diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs index 36956ae4add..b1dcce67742 100644 --- a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -104,6 +104,19 @@ public void AddMessaging_InvalidOptionsFailThroughOptionsContract() Assert.Contains("DurableMessagingOptions validation failed", exception.Message, StringComparison.Ordinal); } + [Fact] + public void AddMessaging_NullDeliveryResiliencePipelineFailsThroughOptionsContract() + { + var services = new ServiceCollection(); + services.AddMessaging(options => options.DeliveryResiliencePipeline = null!); + using var provider = services.BuildServiceProvider(); + + var exception = Assert.Throws( + () => provider.GetRequiredService>().Value); + + Assert.Contains("DurableMessagingOptions validation failed", exception.Message, StringComparison.Ordinal); + } + [Fact] public async Task AddMessaging_BindsInboxContractToDeliveredInboxOverEarlierRegistration() { From 34fd47c7d2334040146cea85166a3a01b335dc5b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:32:57 -0700 Subject: [PATCH 58/95] chore(messaging): remove unused ObjectPool package reference Orleans.Messaging does not use Microsoft.Extensions.ObjectPool. --- src/Orleans.Messaging/Orleans.Messaging.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Orleans.Messaging/Orleans.Messaging.csproj b/src/Orleans.Messaging/Orleans.Messaging.csproj index 0ab2b668c47..7c661000a55 100644 --- a/src/Orleans.Messaging/Orleans.Messaging.csproj +++ b/src/Orleans.Messaging/Orleans.Messaging.csproj @@ -24,7 +24,6 @@ - From ccc8d836021aba40d8220bea4e05b132028316be Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:33:14 -0700 Subject: [PATCH 59/95] test(messaging): fix test helper edge cases RecordingJournalStorageProvider returns zero occurrences for an empty search value instead of looping. MessagingMetricProbe accumulates inbox and outbox depth deltas from the UpDownCounters. TestApplicationProtocol.With preserves Envelope.DeliverAt. --- .../Support/MessagingMetricProbe.cs | 11 ++++------- .../Support/RecordingJournalStorageProvider.cs | 5 +++++ .../Support/TestApplicationProtocol.cs | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs index c07cf9cc4a4..defbd51d690 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs @@ -7,7 +7,7 @@ namespace Orleans.Messaging.Tests.Support; public sealed class MessagingMetricProbe : IDisposable { private readonly ConcurrentDictionary<(string Instrument, string TagValue), long> _measurements = []; - private readonly ConcurrentDictionary _gauges = []; + private readonly ConcurrentDictionary _depths = []; private readonly object _lock = new(); private TaskCompletionSource _changed = CreateSignal(); private readonly MeterListener _listener; @@ -39,11 +39,7 @@ public MessagingMetricProbe() public long GetCount(string instrument, string tagValue = "") => _measurements.TryGetValue((instrument, tagValue), out var count) ? count : 0; - public long GetDepth(string instrument) - { - _listener.RecordObservableInstruments(); - return _gauges.TryGetValue(instrument, out var value) ? value : 0; - } + public long GetDepth(string instrument) => _depths.TryGetValue(instrument, out var value) ? value : 0; public Task WaitForCountAsync( string instrument, @@ -90,7 +86,8 @@ private void OnMeasurement( { if (instrument.Name is "orleans-messaging-inbox-depth" or "orleans-messaging-outbox-depth") { - _gauges[instrument.Name] = measurement; + // The depth instruments are UpDownCounters, so each measurement is a delta. + _depths.AddOrUpdate(instrument.Name, measurement, (_, current) => current + measurement); return; } diff --git a/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs index ada31bd7610..9797fd59255 100644 --- a/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs +++ b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs @@ -43,6 +43,11 @@ public int CountPayloadOccurrences(JournalId journalId, byte[] payload) private static int CountOccurrences(ReadOnlySpan data, ReadOnlySpan value) { + if (value.IsEmpty) + { + return 0; + } + var count = 0; for (var index = data.IndexOf(value); index >= 0; index = data.IndexOf(value)) { diff --git a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs index 90927f17c60..6203391ab19 100644 --- a/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs +++ b/test/Orleans.Messaging.Tests/Support/TestApplicationProtocol.cs @@ -96,7 +96,8 @@ public static InboxMessage With(InboxMessage message, SerializerSessionPool sess MessageId = messageId ?? message.Envelope.MessageId, Payload = payload ?? message.Envelope.Payload, Subject = removeSubject ? null : subject ?? message.Envelope.Subject, - Headers = headers + Headers = headers, + DeliverAt = message.Envelope.DeliverAt } }; } From 3812bb399fff248bc0b02c9f42950568ace7d8c1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 22:33:26 -0700 Subject: [PATCH 60/95] test(messaging): build the messaging test cluster during fixture initialization MessagingClusterFixture now builds its cluster in InitializeAsync, after derived constructors run, so the ConfigureServices and ConfigureOptions overrides are never invoked from the base constructor. Cluster throws until the fixture is initialized. --- .../Support/MessagingClusterFixture.cs | 59 ++++++++++++++----- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index a083266c568..a14282c4e5d 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -12,8 +12,20 @@ namespace Orleans.Messaging.Tests.Support; +/// +/// Hosts an in-process messaging test cluster. +/// +/// +/// The fixture builds and deploys the cluster in , after every derived constructor has run, so +/// and overrides observe fully constructed derived state. +/// public class MessagingClusterFixture : IAsyncLifetime { + private readonly int _initialSilos; + private readonly bool _receiverOnly; + private readonly bool _useServiceCollection; + private InProcessTestCluster? _cluster; + public MessagingClusterFixture() : this(receiverOnly: true) { } @@ -31,9 +43,28 @@ protected MessagingClusterFixture(int initialSilos, bool receiverOnly = false, b HandlerProbe = new HandlerProbe(); SnapshotProbe = new SnapshotProbe(); ActivationProbe = new ActivationValidationProbe(); + _initialSilos = initialSilos; + _receiverOnly = receiverOnly; + _useServiceCollection = useServiceCollection; + } + + public InProcessTestCluster Cluster => + _cluster ?? throw new InvalidOperationException($"The cluster is available after {nameof(InitializeAsync)} builds it."); + + public IClusterClient Client => Cluster.Client!; + public FakeTimeProvider Clock { get; } + public ControlledJournalStorageProvider Storage { get; } + public MessagingMetricProbe Metrics { get; } + public DurableJobManagerProbe JobManagerProbe { get; } + public HandlerProbe HandlerProbe { get; } + public SnapshotProbe SnapshotProbe { get; } + public ActivationValidationProbe ActivationProbe { get; } + + private InProcessTestCluster BuildCluster() + { var clusterId = $"messaging-{Guid.NewGuid():N}"; var serviceId = $"messaging-service-{Guid.NewGuid():N}"; - var builder = new InProcessTestClusterBuilder((short)initialSilos); + var builder = new InProcessTestClusterBuilder((short)_initialSilos); builder.ConfigureClient(clientBuilder => clientBuilder.Configure(options => { @@ -60,11 +91,11 @@ protected MessagingClusterFixture(int initialSilos, bool receiverOnly = false, b siloBuilder.UseInMemoryDurableJobs(); // The test grains journal Orleans-serialized types. siloBuilder.Services.Configure(options => options.JournalFormatKey = "orleans-binary"); - if (receiverOnly) + if (_receiverOnly) { ReceiverTestServices.Add(siloBuilder.Services, ConfigureOptions); } - else if (useServiceCollection) + else if (_useServiceCollection) { siloBuilder.Services.AddMessaging(ConfigureOptions); } @@ -86,7 +117,7 @@ protected MessagingClusterFixture(int initialSilos, bool receiverOnly = false, b serviceProvider => (IJournalStorageCatalog)serviceProvider.GetRequiredService()); ConfigureServices(siloBuilder.Services); }); - Cluster = builder.Build(); + return builder.Build(); } /// @@ -107,16 +138,6 @@ private sealed class TestJobClock(TimeProvider clock) : TimeProvider public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); } - public InProcessTestCluster Cluster { get; } - public IClusterClient Client => Cluster.Client!; - public FakeTimeProvider Clock { get; } - public ControlledJournalStorageProvider Storage { get; } - public MessagingMetricProbe Metrics { get; } - public DurableJobManagerProbe JobManagerProbe { get; } - public HandlerProbe HandlerProbe { get; } - public SnapshotProbe SnapshotProbe { get; } - public ActivationValidationProbe ActivationProbe { get; } - public Task WaitForEffectCountAsync(IMessagingTestGrain grain, int expected) => SnapshotProbe.WaitAsync( grain.GetGrainId(), @@ -290,10 +311,16 @@ protected virtual void ConfigureOptions(DurableMessagingOptions options) options.OutboxBatchSize = 8; } - public ValueTask InitializeAsync() => new(Cluster.DeployAsync()); + public ValueTask InitializeAsync() + { + if (_cluster is not null) throw new InvalidOperationException("The cluster is already initialized."); + _cluster = BuildCluster(); + return new(_cluster.DeployAsync()); + } + public async ValueTask DisposeAsync() { - await Cluster.DisposeAsync(); + if (_cluster is not null) await _cluster.DisposeAsync(); Metrics.Dispose(); } } From 1631c954e504db9e01be76d7285bda45a6b73743 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 23:40:06 -0700 Subject: [PATCH 61/95] fix(messaging): normalize non-positive work-source delays to a prompt wake-up IInboxWorkSource.GetWakeupDelay permits any non-positive delay, but a large negative delay such as TimeSpan.MinValue produced an unrepresentable due time and failed write preparation. The inbox normalizes non-positive source delays to zero. --- src/Orleans.Messaging/Inbox.cs | 6 +++++- .../Functional/InboxWorkSourceTests.cs | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index c257b91d6be..9b8ad6d26bd 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -428,12 +428,16 @@ internal bool RemoveDeadLetter(MessageId messageId) return next; } + // Work sources may report any non-positive delay to request a prompt wake-up; it is normalized to zero so that due + // times derived from it stay representable. private TimeSpan? GetWorkSourceDelay() { TimeSpan? next = null; foreach (var source in _workSources) { - if (source.GetWakeupDelay() is { } delay && (next is null || delay < next)) next = delay; + if (source.GetWakeupDelay() is not { } delay) continue; + if (delay <= TimeSpan.Zero) return TimeSpan.Zero; + if (next is null || delay < next) next = delay; } return next; diff --git a/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs index f58c8354e4e..eca2362bd90 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxWorkSourceTests.cs @@ -34,6 +34,23 @@ public async Task WriteWhichMakesWorkPending_SchedulesTheWakeupBeforeTheWriteIsS await grain.CompleteWorkAsync(); } + [Fact] + public async Task NegativeSourceDelay_SchedulesAPromptWakeupBeforeTheWriteIsStored() + { + var grain = NewWorkGrain(); + await grain.GetPendingAsync(); + using var write = Fixture.Storage.BlockWrite(Journal(grain)); + + var adding = grain.AddWorkAsync(TimeSpan.MinValue); + await write.WaitUntilEnteredAsync(); + + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(MessagingPump.JobName, grain.GetGrainId())); + write.Release(); + await adding; + Assert.Equal(1, await grain.GetPendingAsync()); + await grain.CompleteWorkAsync(); + } + [Fact] public async Task CrashAfterTheWriteWhichMakesWorkPending_ReactivatesTheGrain() { From 68101ebd58cf515b0034c344a982b0bf434ea3c3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 18 Oct 2026 00:15:04 -0700 Subject: [PATCH 62/95] fix(messaging): answer duplicates of sink-rejected messages with their rejection An inbox sink's rejection completes the message, so a redelivery within DeduplicationWindow was answered Duplicate, which the sender's outbox retires as Delivered. A sender whose first answer was lost, such as one that deactivated before journaling the retirement, therefore retired a rejected message as delivered. The processed record now keeps the sink's reason and code, through journal entries and snapshots, so the inbox answers duplicates with the original DeadLettered result while the record is retained, independently of the dead letter's retention. --- .../Configuration/DurableMessagingOptions.cs | 3 +- src/Orleans.Messaging/IInboxMessageSink.cs | 3 +- src/Orleans.Messaging/Inbox.State.cs | 36 +++++++++++++++++-- src/Orleans.Messaging/Inbox.cs | 17 +++++---- src/Orleans.Messaging/README.md | 6 +++- .../MessagingCommandSerializationTests.cs | 8 ++++- .../Functional/MessageSourceAndSinkTests.cs | 22 ++++++++++-- 7 files changed, 80 insertions(+), 15 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index df88f501b39..37982d12a54 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -55,7 +55,8 @@ public class DurableMessagingOptions /// /// Gets or sets the time window for tracking processed messages to prevent duplicates. - /// Messages that were processed within this window will be rejected with DeliveryResult.Duplicate(). + /// A message processed within this window is answered with DeliveryResult.Duplicate(), or, when an inbox + /// sink rejected it on delivery, with the same DeliveryResult.DeadLettered reason and code. /// /// /// diff --git a/src/Orleans.Messaging/IInboxMessageSink.cs b/src/Orleans.Messaging/IInboxMessageSink.cs index 27087c3db15..fd633303641 100644 --- a/src/Orleans.Messaging/IInboxMessageSink.cs +++ b/src/Orleans.Messaging/IInboxMessageSink.cs @@ -57,7 +57,8 @@ public interface IInboxMessageSink /// /// /// : the sink rejects the message and stages nothing. The inbox - /// dead-letters it with its payload and the result's reason, and records it as processed. + /// dead-letters it with its payload and the result's reason, and records it as processed. The processed record keeps + /// the reason, so a duplicate of the message, such as a retry whose first answer was lost, receives the same rejection. /// /// /// diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index c7c419a0aff..7b113143f85 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -12,6 +12,9 @@ internal sealed partial class Inbox { private readonly Dictionary _messages = []; private readonly Dictionary _processed = []; + + // The rejections which sinks reported on delivery, by processed message, so a duplicate receives the same answer. + private readonly Dictionary _rejections = []; private readonly Dictionary _deadLetters = []; private JournalStreamWriter _writer; @@ -77,6 +80,7 @@ void IStateMachine.Reset(JournalStreamWriter writer) _writer = writer; _messages.Clear(); _processed.Clear(); + _rejections.Clear(); _deadLetters.Clear(); _scheduledCount = 0; _stagedWakeup = null; @@ -117,7 +121,11 @@ void IStateMachine.WriteSnapshot(JournalStreamWriter writer) var now = TimeProvider.GetUtcNow(); foreach (var (key, processedAt) in _processed) { - if (MessagingTime.IsExpired(now, processedAt, Options.DeduplicationWindow)) _processed.Remove(key); + if (MessagingTime.IsExpired(now, processedAt, Options.DeduplicationWindow)) + { + _processed.Remove(key); + _rejections.Remove(key); + } } Shared.InboxCodec.WriteSet(new() @@ -126,7 +134,12 @@ void IStateMachine.WriteSnapshot(JournalStreamWriter writer) Snapshot = new() { Messages = _messages.Values.ToArray(), - Processed = _processed.Select(static pair => new InboxProcessedEntry { MessageId = pair.Key, ProcessedAt = pair.Value }).ToArray(), + Processed = _processed.Select(pair => new InboxProcessedEntry + { + MessageId = pair.Key, + ProcessedAt = pair.Value, + Rejection = _rejections.GetValueOrDefault(pair.Key) + }).ToArray(), DeadLetters = _deadLetters.Values.ToArray() } }, writer); @@ -143,6 +156,7 @@ private void Apply(InboxCommand command) var message = command.Message ?? throw new InvalidOperationException("An accept command requires its message."); _messages.Add(message.Envelope.MessageId, new InboxEntry { Message = message, State = new(), Sequence = ++_lastSequence }); _processed.Remove(message.Envelope.MessageId); + _rejections.Remove(message.Envelope.MessageId); if (message.Envelope.DeliverAt is not null) _scheduledCount++; ReportDepth(1); break; @@ -158,6 +172,7 @@ private void Apply(InboxCommand command) _processed[command.MessageId] = command.ProcessedAt; _lastCompletionSequence = ++_lastSequence; + if (command.Rejection is { } rejection) _rejections[command.MessageId] = rejection; if (command.DeadLetter is { } letter) _deadLetters[command.MessageId] = letter; break; case InboxCommandKind.RemoveDeadLetters: @@ -168,6 +183,7 @@ private void Apply(InboxCommand command) var previousCount = _messages.Count; _messages.Clear(); _processed.Clear(); + _rejections.Clear(); _deadLetters.Clear(); _scheduledCount = 0; foreach (var restored in snapshot.Messages) @@ -178,7 +194,12 @@ private void Apply(InboxCommand command) ReportDepth(_messages.Count - previousCount); - foreach (var processed in snapshot.Processed) _processed.Add(processed.MessageId, processed.ProcessedAt); + foreach (var processed in snapshot.Processed) + { + _processed.Add(processed.MessageId, processed.ProcessedAt); + if (processed.Rejection is { } restoredRejection) _rejections.Add(processed.MessageId, restoredRejection); + } + foreach (var restored in snapshot.DeadLetters) _deadLetters.Add(restored.Message.Envelope.MessageId, restored); break; default: @@ -207,6 +228,7 @@ internal sealed class InboxCommand [Id(5)] public InboxDeadLetter? DeadLetter { get; init; } [Id(6)] public MessageId[]? RemovedKeys { get; init; } [Id(7)] public InboxSnapshot? Snapshot { get; init; } + [Id(8)] public InboxRejection? Rejection { get; init; } } [GenerateSerializer, Alias("Orleans.Messaging.InboxSnapshot")] @@ -222,6 +244,14 @@ internal sealed class InboxProcessedEntry { [Id(0)] public MessageId MessageId { get; init; } [Id(1)] public DateTimeOffset ProcessedAt { get; init; } + [Id(2)] public InboxRejection? Rejection { get; init; } +} + +/// The answer which an inbox sink gave when it rejected a message on delivery. +[GenerateSerializer, Alias("Orleans.Messaging.InboxRejection")] +internal sealed class InboxRejection +{ + [Id(0)] public required string Reason { get; init; } } [GenerateSerializer, Alias("Orleans.Messaging.InboxEntry")] diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 9b8ad6d26bd..c856d43aa7d 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -91,7 +91,11 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel if (pending is not null) _pump.Signal(); Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); - return DeliveryResult.Duplicate(); + + // A sender whose answer was lost delivers again, so it receives the rejection which a sink first answered. + return pending is null && _rejections.TryGetValue(key, out var rejection) + ? DeliveryResult.DeadLettered(rejection.Reason) + : DeliveryResult.Duplicate(); } // Sink-owned messages never occupy inbox capacity and need no handler. A sink's duplicate can still be staged, @@ -176,7 +180,7 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel else { LogSinkDeadLettered(Logger, error, key, GrainContext.GrainId, reason); - StageDeadLetter(message, reason, attemptCount: 1); + StageDeadLetter(message, reason, attemptCount: 1, new() { Reason = reason }); } } catch (Exception exception) @@ -356,16 +360,17 @@ private async ValueTask ProcessAsync(InboxEntry entry, CancellationToken cancell private bool IsCurrent(InboxEntry entry) => _messages.TryGetValue(entry.Message.Envelope.MessageId, out var current) && ReferenceEquals(current, entry); - private void StageCompletion(MessageId messageId, InboxDeadLetter? deadLetter) => + private void StageCompletion(MessageId messageId, InboxDeadLetter? deadLetter, InboxRejection? rejection = null) => Emit(new() { Kind = InboxCommandKind.Complete, MessageId = messageId, ProcessedAt = TimeProvider.GetUtcNow(), - DeadLetter = deadLetter + DeadLetter = deadLetter, + Rejection = rejection }); - private void StageDeadLetter(InboxMessage message, string reason, int attemptCount) + private void StageDeadLetter(InboxMessage message, string reason, int attemptCount, InboxRejection? rejection = null) { var now = TimeProvider.GetUtcNow(); var key = message.Envelope.MessageId; @@ -376,7 +381,7 @@ private void StageDeadLetter(InboxMessage message, string reason, int attemptCou DeadLetteredAt = now, Reason = reason, AttemptCount = attemptCount - }); + }, rejection); } private bool CompactDeadLetters(DateTimeOffset now, int reservedCapacity = 0) diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 0e59ee35f7e..2c5382dc857 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -115,7 +115,9 @@ construction or activation setup, before journal recovery. - `Backpressured`: the sink is at capacity, so the sender retries. A self-sent message fails its `Send` instead. - `DeadLettered`: the inbox dead-letters the message with the sink's reason. A - sink that throws or reports another status is treated the same way. + sink that throws or reports another status is treated the same way. The processed + record keeps the reason, so a duplicate within `DeduplicationWindow`, such as a + retry whose first answer was lost, receives the same rejection. Sink-owned messages never occupy capacity and have no inbox retry lifecycle; the sink honors their `DeliverAt`. Declined messages follow the handler path once @@ -137,6 +139,8 @@ receiver returns a `DeliveryResult`: - `Accepted`: the message is durable in the inbox, or, for a message an inbox sink accepted, in the sink's state together with its processed record. - `Duplicate`: the message ID is pending or completed within `DeduplicationWindow`. + A completed message which an inbox sink rejected is answered with its original + `DeadLettered` result instead. - `Backpressured`: the inbox holds `MaxCapacity` pending messages without `DeliverAt`. The inbox admits scheduled messages regardless of `MaxCapacity`. - `HandlerNotFound`: the receiver has no registered handlers. diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index b63f3a36647..2d1bd13ff03 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -69,12 +69,14 @@ public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWh private static InboxCommand[] CreateInboxCommands() { var deadLetter = new InboxDeadLetter { Message = InboxMessage("dead"), DeadLetteredAt = Now, Reason = "failed", AttemptCount = 3 }; + var rejection = new InboxRejection { Reason = "failed" }; return [ new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("accept"), State = new() }, new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("scheduled", Now), State = new() }, new() { Kind = InboxCommandKind.Retry, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom" } }, new() { Kind = InboxCommandKind.Complete, MessageId = MessageId.Create("orders", "done"), ProcessedAt = Now, DeadLetter = deadLetter }, + new() { Kind = InboxCommandKind.Complete, MessageId = MessageId.Create("orders", "dead"), ProcessedAt = Now, DeadLetter = deadLetter, Rejection = rejection }, new() { Kind = InboxCommandKind.RemoveDeadLetters, RemovedKeys = [MessageId.Create("orders", "dead")] }, new() { @@ -82,7 +84,11 @@ private static InboxCommand[] CreateInboxCommands() Snapshot = new() { Messages = [new() { Message = InboxMessage("pending"), State = new() { AttemptCount = 1 } }, new() { Message = InboxMessage("scheduled", Now), State = new() }], - Processed = [new() { MessageId = MessageId.Create("orders", "done"), ProcessedAt = Now }], + Processed = + [ + new() { MessageId = MessageId.Create("orders", "done"), ProcessedAt = Now }, + new() { MessageId = MessageId.Create("orders", "dead"), ProcessedAt = Now, Rejection = rejection } + ], DeadLetters = [deadLetter] } } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 23a2a3a1e7a..60542564091 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -51,7 +51,7 @@ public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndTheSinkReportsLat [InlineData(SinkMode.Throw, SourceSinkGrain.Rejection)] [InlineData(SinkMode.DeadLetter, SourceSinkGrain.Rejection)] [InlineData(SinkMode.UnsupportedStatus, "The inbox message sink reported the unsupported delivery status 'HandlerNotFound'.")] - public async Task SinkRejection_DeadLettersTheMessageAndRecordsItAsProcessed(SinkMode mode, string reason) + public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWithTheSameRejection(SinkMode mode, string reason) { var receiver = NewGrain(); Probe.For(receiver).Mode = mode; @@ -80,10 +80,28 @@ public async Task SinkRejection_DeadLettersTheMessageAndRecordsItAsProcessed(Sin Assert.Equal(1, letter.AttemptCount); Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + // A duplicate receives the rejection from the processed record, which outlives the dead letter, without + // reaching the sink. Probe.For(receiver).Mode = SinkMode.Accept; - Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(message, Token)).Status); + AssertRejected(await extension.DeliverAsync(message, Token)); + Assert.True(await OnGrainAsync(receiver, context => + context.ActivationServices.GetRequiredService().RemoveInboxDeadLetter(message.Envelope.MessageId))); + AssertRejected(await extension.DeliverAsync(message, Token)); + Assert.Equal(1, Probe.For(receiver).SinkOffers); Assert.Equal(0, Probe.For(receiver).SinkAcceptances); Assert.Equal(0, Probe.For(receiver).Handled); + + // Once the processed record leaves the deduplication window, the message is offered to the sink again. + fixture.Clock.Advance(MessageSourceAndSinkFixture.DeduplicationWindow); + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(2, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + + void AssertRejected(DeliveryResult duplicate) + { + Assert.Equal(DeliveryStatus.DeadLettered, duplicate.Status); + Assert.Equal(reason, duplicate.Message); + } } [Fact] From b0682139b61f0c0c3d31c622faa893721cd8dafd Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 15:29:19 -0700 Subject: [PATCH 63/95] feat(messaging): deliver outbox messages --- .../Hosting/MessagingExtensions.cs | 7 +- .../IMessagingDiagnostics.cs | 28 +- src/Orleans.Messaging/MessageState.cs | 32 ++ .../MessagingGrainTypeConfigurator.cs | 1 + src/Orleans.Messaging/MessagingInstruments.cs | 20 + src/Orleans.Messaging/MessagingPump.cs | 32 +- src/Orleans.Messaging/MessagingShared.cs | 9 + src/Orleans.Messaging/MessagingStateNames.cs | 1 + src/Orleans.Messaging/Outbox.State.cs | 148 ++++++ src/Orleans.Messaging/Outbox.cs | 434 ++++++++++++++++ src/Orleans.Messaging/README.md | 63 ++- .../Orleans.Messaging/Orleans.Messaging.cs | 3 + .../Contracts/EnvelopeContractTests.cs | 1 + .../MessagingCommandSerializationTests.cs | 41 ++ .../Functional/JsonJournalMessagingTests.cs | 176 +++++++ .../MessagingActivationBehaviorTests.cs | 48 ++ .../MessagingGrainTypeConfiguratorTests.cs | 487 ++++++++++++++++++ .../MessagingMetricCardinalityTests.cs | 16 +- .../MessagingProviderCutoverTests.cs | 87 ++++ .../MultiSiloMessagingFailoverTests.cs | 64 +++ .../Functional/OutboxDeliveryBehaviorTests.cs | 241 +++++++++ .../Functional/OutboxJournalRecoveryTests.cs | 149 ++++++ .../PublicInboxHandlerTransactionTests.cs | 314 +++++++++++ ...ublicMessagingRoutingAndDeadLetterTests.cs | 67 +++ ...viceCollectionMessagingCompositionTests.cs | 11 + .../PublicMessagingRegistrationTests.cs | 2 +- .../Support/BootstrapOutboxServices.cs | 133 +++++ .../Support/BootstrapTestGrains.cs | 321 ++++++++++++ .../Support/InboxTestAccess.cs | 2 + .../Support/MessagingTestGrains.cs | 22 +- .../Support/OutboxStateTestAccess.cs | 11 + .../Support/OutboxTestMessages.cs | 31 ++ .../Support/RawPayloadTestGrain.cs | 115 +++++ 33 files changed, 3073 insertions(+), 44 deletions(-) create mode 100644 src/Orleans.Messaging/Outbox.State.cs create mode 100644 src/Orleans.Messaging/Outbox.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingActivationBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/OutboxJournalRecoveryTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/PublicServiceCollectionMessagingCompositionTests.cs create mode 100644 test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs create mode 100644 test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs create mode 100644 test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs create mode 100644 test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs create mode 100644 test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs index 2122ffe224c..78ef142cbdd 100644 --- a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -17,7 +17,7 @@ namespace Orleans.Hosting; public static class MessagingExtensions { /// - /// Adds inbox messaging support to the silo. + /// Adds inbox and outbox messaging support to the silo. /// /// /// Grains implementing or deriving from @@ -33,7 +33,7 @@ public static ISiloBuilder AddMessaging(this ISiloBuilder builder, Action - /// Adds inbox messaging services. + /// Adds inbox and outbox messaging services. /// /// /// Grains implementing or deriving from @@ -49,6 +49,7 @@ public static IServiceCollection AddMessaging(this IServiceCollection services, services.Configure(options => { options.AddTypeInfoResolver(InboxJsonContext.Default); + options.AddTypeInfoResolver(OutboxJsonContext.Default); }); var optionsBuilder = services.AddOptions(); @@ -77,6 +78,8 @@ public static IServiceCollection AddMessaging(this IServiceCollection services, services.TryAddScoped(); services.TryAddScoped(); services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); + services.TryAddScoped(); + services.TryAddScoped(sp => sp.GetRequiredService()); services.TryAddScoped(); services.TryAddEnumerable( ServiceDescriptor.Singleton()); diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs index 18c56eecb87..00b9c78c22c 100644 --- a/src/Orleans.Messaging/IMessagingDiagnostics.cs +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -15,6 +15,11 @@ public interface IMessagingDiagnostics /// IReadOnlyList> InboxDeadLetters { get; } + /// + /// Gets messages which could not be delivered from the outbox. + /// + IReadOnlyList> OutboxDeadLetters { get; } + /// /// Stages removal of an inbox dead letter. /// @@ -24,6 +29,16 @@ public interface IMessagingDiagnostics /// The removal becomes durable with the grain's next journal write. /// bool RemoveInboxDeadLetter(MessageId messageId); + + /// + /// Stages removal of an outbox dead letter. + /// + /// The message identifier. + /// when the dead letter existed and was removed. + /// + /// The removal becomes durable with the grain's next journal write. + /// + bool RemoveOutboxDeadLetter(MessageId messageId); } /// @@ -62,7 +77,7 @@ public sealed class DeadLetter public int AttemptCount { get; init; } } -internal sealed class MessagingDiagnostics(Inbox inbox) : IMessagingDiagnostics +internal sealed class MessagingDiagnostics(Inbox inbox, Outbox outbox) : IMessagingDiagnostics { public IReadOnlyList> InboxDeadLetters => inbox.DeadLetterState.Values.Select(static entry => new DeadLetter @@ -73,5 +88,16 @@ internal sealed class MessagingDiagnostics(Inbox inbox) : IMessagingDiagnostics AttemptCount = entry.AttemptCount }).ToList(); + public IReadOnlyList> OutboxDeadLetters => + outbox.DeadLetterState.Values.Select(static entry => new DeadLetter + { + Message = entry.Message, + DeadLetteredAt = entry.DeadLetteredAt, + Reason = entry.Reason, + AttemptCount = entry.AttemptCount + }).ToList(); + public bool RemoveInboxDeadLetter(MessageId messageId) => inbox.RemoveDeadLetter(messageId); + + public bool RemoveOutboxDeadLetter(MessageId messageId) => outbox.RemoveDeadLetter(messageId); } diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs index 4294365ce1a..c80ec0b2adb 100644 --- a/src/Orleans.Messaging/MessageState.cs +++ b/src/Orleans.Messaging/MessageState.cs @@ -23,6 +23,22 @@ internal sealed class InboxMessageState public DateTimeOffset? ExpirationTime { get; set; } } +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessageState")] +internal sealed class OutboxMessageState +{ + [Id(0)] + public int AttemptCount { get; set; } + + [Id(1)] + public DateTimeOffset? NextAttemptAt { get; set; } + + [Id(2)] + public string? LastError { get; set; } + + [Id(3)] + public DateTimeOffset? EnqueuedAt { get; set; } +} + [GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] internal sealed class InboxDeadLetter { @@ -38,3 +54,19 @@ internal sealed class InboxDeadLetter [Id(3)] public int AttemptCount { get; init; } } + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxDeadLetter")] +internal sealed class OutboxDeadLetter +{ + [Id(0)] + public required OutboxMessage Message { get; init; } + + [Id(1)] + public required DateTimeOffset DeadLetteredAt { get; init; } + + [Id(2)] + public required string Reason { get; init; } + + [Id(3)] + public int AttemptCount { get; init; } +} diff --git a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs index 4f0d23c43fd..b67b53ff0f9 100644 --- a/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs +++ b/src/Orleans.Messaging/MessagingGrainTypeConfigurator.cs @@ -20,6 +20,7 @@ public void Configure(GrainType grainType, GrainProperties properties, GrainType MessagingActivationValidator.Validate(grainClass, placementStrategy); var activationServices = context.ActivationServices; var inbox = activationServices.GetRequiredService(); + _ = activationServices.GetRequiredService(); context.GetComponent()!.GetOrSetExtension(() => inbox); }); } diff --git a/src/Orleans.Messaging/MessagingInstruments.cs b/src/Orleans.Messaging/MessagingInstruments.cs index 852a23a68ec..e6aba94026a 100644 --- a/src/Orleans.Messaging/MessagingInstruments.cs +++ b/src/Orleans.Messaging/MessagingInstruments.cs @@ -13,13 +13,19 @@ internal sealed class MessagingInstruments(OrleansInstruments instruments) private readonly Counter _inboxMessagesReceived = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-received"); private readonly Counter _inboxMessagesProcessed = instruments.Meter.CreateCounter("orleans-messaging-inbox-messages-processed"); + private readonly Counter _outboxMessagesSent = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-sent"); + private readonly Counter _outboxMessagesDelivered = instruments.Meter.CreateCounter("orleans-messaging-outbox-messages-delivered"); private readonly Histogram _inboxProcessingDuration = instruments.Meter.CreateHistogram("orleans-messaging-inbox-processing-duration", MillisecondsUnit); + private readonly Histogram _outboxDeliveryDuration = instruments.Meter.CreateHistogram("orleans-messaging-outbox-delivery-duration", MillisecondsUnit); private readonly UpDownCounter _inboxDepth = instruments.Meter.CreateUpDownCounter("orleans-messaging-inbox-depth"); + private readonly UpDownCounter _outboxDepth = instruments.Meter.CreateUpDownCounter("orleans-messaging-outbox-depth"); internal static MessagingInstruments CreateForDirectConstruction() => new(new OrleansInstruments(new DirectMeterFactory())); internal void OnInboxDepthChanged(string grainType, int delta) => AddDepth(_inboxDepth, grainType, delta); + internal void OnOutboxDepthChanged(string grainType, int delta) => AddDepth(_outboxDepth, grainType, delta); + private static void AddDepth(UpDownCounter counter, string grainType, int delta) { if (delta != 0 && counter.Enabled) @@ -37,6 +43,20 @@ internal void OnInboxMessageProcessed(string grainType, string status) => internal void OnInboxProcessingDuration(TimeSpan duration, string grainType) => Record(_inboxProcessingDuration, duration, grainType); + internal void OnOutboxMessageSent(string grainType) + { + if (_outboxMessagesSent.Enabled) + { + _outboxMessagesSent.Add(1, new KeyValuePair(GrainTypeTagName, grainType)); + } + } + + internal void OnOutboxMessageDelivered(string grainType, string status) => + Add(_outboxMessagesDelivered, grainType, status); + + internal void OnOutboxDeliveryDuration(TimeSpan duration, string grainType) => + Record(_outboxDeliveryDuration, duration, grainType); + private static void Add(Counter counter, string grainType, string status) { if (counter.Enabled) diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index c09d8adbffb..4d180561dcc 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -9,11 +9,11 @@ namespace Orleans.Messaging; /// -/// Drives inbox processing for one activation. +/// Drives inbox processing and outbox delivery for one activation. /// /// -/// A single loop runs on the activation scheduler and services the inbox whenever it is signaled. -/// One durable job keeps waking the grain while the inbox or a registered holds work +/// A single loop runs on the activation scheduler and services both queues whenever it is signaled. +/// One durable job keeps waking the grain while either queue or a registered holds work /// which the activation can perform; inbox messages wait for a handler without a wake-up. /// Work staged with an earlier due time than the current wake-up schedules an earlier one, and the job which a newer, /// earlier wake-up supersedes completes when it runs. @@ -29,6 +29,7 @@ internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecy private readonly SingleWaiterAutoResetEvent _signal = new() { RunContinuationsAsynchronously = true }; private readonly CancellationTokenSource _shutdown = new(); private Inbox? _inbox; + private Outbox? _outbox; private Task? _loop; private TaskCompletionSource? _nextPass; private Task? _scheduling; @@ -57,6 +58,7 @@ public MessagingPump(IGrainContext grainContext, IJournaledStateManager stateMan grainContext.ObservableLifecycle.Subscribe(nameof(MessagingPump), GrainLifecycleStage.SetupState, static _ => Task.CompletedTask, _ => { _inbox?.ReleaseDepth(); + _outbox?.ReleaseDepth(); return Task.CompletedTask; }); } @@ -72,8 +74,12 @@ public MessagingPump(IGrainContext grainContext, IJournaledStateManager stateMan internal void Register(Inbox inbox) => _inbox = inbox; + internal void Register(Outbox outbox) => _outbox = outbox; + private Inbox Inbox => _inbox ?? throw new InvalidOperationException("The messaging inbox has not been registered."); + private Outbox Outbox => _outbox ?? throw new InvalidOperationException("The messaging outbox has not been registered."); + public bool CanHandle(string jobName) => string.Equals(jobName, JobName, StringComparison.Ordinal); /// Requests a processing pass. @@ -139,7 +145,7 @@ public async ValueTask EnsureWakeupAsync(DateTimeOffset dueTime, CancellationTok public async ValueTask ExecuteJobAsync(IJobRunContext context, CancellationToken cancellationToken) { - // A running job keeps rescheduling itself until the inbox and every work source are empty. + // A running job keeps rescheduling itself until both queues and every work source are empty. var jobId = context.Job.Id; if (_wakeupJobId is null) { @@ -157,7 +163,7 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte { // This attempt no longer guarantees a future wake-up. CompleteWakeup(jobId); - if (!_shutdown.IsCancellationRequested && Inbox.HasWork) + if (!_shutdown.IsCancellationRequested && (Inbox.HasWork || Outbox.Count > 0)) { RepairWakeupAsync().Ignore(); } @@ -166,12 +172,15 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte } var now = Shared.JobTimeProvider.GetUtcNow(); - if (Inbox.GetNextWakeupDelay(now) is not { } delay) + var inboxDelay = Inbox.GetNextWakeupDelay(now); + var outboxDelay = Outbox.GetNextWakeupDelay(); + if (inboxDelay is null && outboxDelay is null) { CompleteWakeup(jobId); return DurableJobRunResult.Completed; } + var delay = Min(inboxDelay, outboxDelay); var next = MessagingTime.AddClamped(now, delay > TimeSpan.Zero ? delay : RecheckDelay); if (_wakeupJobId is not null && _wakeupJobId != jobId && _wakeupDueTime <= next) { @@ -191,6 +200,9 @@ private void CompleteWakeup(string jobId) _wakeupEpoch++; } + private static TimeSpan Min(TimeSpan? left, TimeSpan? right) => + left is { } l ? right is { } r && r < l ? r : l : right!.Value; + private async Task RepairWakeupAsync() { try @@ -219,6 +231,7 @@ private async Task RunAsync() try { progress = await Inbox.ProcessReadyAsync(cancellationToken).ConfigureAwait(true); + progress |= await Outbox.AdvanceAsync(cancellationToken).ConfigureAwait(true); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -259,8 +272,9 @@ private void SetKeepAlive(bool keepAlive) public async Task OnStart(CancellationToken cancellationToken) { + await Outbox.StartAsync(cancellationToken).ConfigureAwait(true); await Inbox.StartAsync(cancellationToken).ConfigureAwait(true); - if (Inbox.HasWork) + if (Inbox.HasWork || Outbox.Count > 0) { await EnsureWakeupAsync(Shared.JobTimeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(true); } @@ -271,7 +285,7 @@ public async Task OnStop(CancellationToken cancellationToken) var scheduling = _scheduling; Stop(); - // Grain calls finish before deactivation stops the lifecycle, so the loop owns every remaining write. + // Grain calls finish before deactivation stops the lifecycle, so the loop and outbox deliveries own every remaining write. if (_loop is { } loop) { await loop.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); @@ -281,6 +295,8 @@ public async Task OnStop(CancellationToken cancellationToken) { await ((Task)scheduling).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); } + + await Outbox.StopAsync().ConfigureAwait(true); } /// Stops processing for this activation. diff --git a/src/Orleans.Messaging/MessagingShared.cs b/src/Orleans.Messaging/MessagingShared.cs index c76340e45eb..8362460eb56 100644 --- a/src/Orleans.Messaging/MessagingShared.cs +++ b/src/Orleans.Messaging/MessagingShared.cs @@ -25,12 +25,15 @@ public MessagingShared(IServiceProvider services, GrainType grainType) TimeProvider = services.GetRequiredService(); JobTimeProvider = services.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs); JobManager = services.GetRequiredService(); + GrainFactory = services.GetRequiredService(); Instruments = services.GetRequiredService(); var loggers = services.GetRequiredService(); InboxLogger = loggers.CreateLogger(); + OutboxLogger = loggers.CreateLogger(); PumpLogger = loggers.CreateLogger(); var format = services.GetRequiredService>().Value.JournalFormatKey; InboxCodec = services.GetRequiredKeyedService>(format); + OutboxCodec = services.GetRequiredKeyedService>(format); } /// Gets the grain type metric tag. @@ -45,11 +48,17 @@ public MessagingShared(IServiceProvider services, GrainType grainType) public ILocalDurableJobManager JobManager { get; } + public IGrainFactory GrainFactory { get; } + public MessagingInstruments Instruments { get; } public ILogger InboxLogger { get; } + public ILogger OutboxLogger { get; } + public ILogger PumpLogger { get; } public IDurableValueCommandCodec InboxCodec { get; } + + public IDurableValueCommandCodec OutboxCodec { get; } } diff --git a/src/Orleans.Messaging/MessagingStateNames.cs b/src/Orleans.Messaging/MessagingStateNames.cs index 1b947f730d9..d9cd0bb37c9 100644 --- a/src/Orleans.Messaging/MessagingStateNames.cs +++ b/src/Orleans.Messaging/MessagingStateNames.cs @@ -5,4 +5,5 @@ internal static class MessagingStateNames private const string Prefix = "orleans.messaging."; public const string Inbox = Prefix + "inbox"; + public const string Outbox = Prefix + "outbox"; } diff --git a/src/Orleans.Messaging/Outbox.State.cs b/src/Orleans.Messaging/Outbox.State.cs new file mode 100644 index 00000000000..1c0a048831b --- /dev/null +++ b/src/Orleans.Messaging/Outbox.State.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Journaling; + +namespace Orleans.Messaging; + +internal sealed partial class Outbox +{ + // Entries are durable once a completed write captured their sequence number. + private long _lastSequence; + private long _capturedSequence; + private long _durableSequence; + + ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) => + _lastSequence != _capturedSequence ? _pump.EnsureWakeupAsync(cancellationToken) : ValueTask.CompletedTask; + + private void OnWriteCaptured() => _capturedSequence = _lastSequence; + + void IStateMachine.OnWriteCompleted() + { + _durableSequence = _capturedSequence; + _pump.Signal(); + } + + private OutboxCommand CreateSnapshot() => new() + { + Kind = OutboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = _messages.Values.ToArray(), + DeadLetters = _deadLetters.Values.ToArray() + } + }; + + private void Emit(OutboxCommand command) + { + Shared.OutboxCodec.WriteSet(command, _writer); + Apply(command); + } + + void IStateMachine.Reset(JournalStreamWriter writer) + { + _writer = writer; + ResetState(); + } + + void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, Shared.OutboxCodec).Apply(entry.Reader, this); + + void IStateMachine.OnRecoveryCompleted() => OnRecoveryCompleted(); + void IStateMachine.WritePendingEntries(JournalStreamWriter writer) => OnWriteCaptured(); + + void IStateMachine.WriteSnapshot(JournalStreamWriter writer) + { + Shared.OutboxCodec.WriteSet(CreateSnapshot(), writer); + OnWriteCaptured(); + } + + void IDurableValueCommandHandler.ApplySet(OutboxCommand command) => Apply(command); + + private void Apply(OutboxCommand command) + { + switch (command.Kind) + { + case OutboxCommandKind.Enqueue: + var message = command.Message ?? throw new InvalidOperationException("An enqueue command requires its message."); + var state = command.State ?? throw new InvalidOperationException("An enqueue command requires its delivery state."); + _messages.Add(message.Envelope.MessageId, new OutboxEntry { Message = message, State = state, Sequence = ++_lastSequence }); + ReportDepth(1); + break; + case OutboxCommandKind.Delivery: + if (command.State is { } retry) _messages[command.MessageId].State = retry; + else if (_messages.Remove(command.MessageId)) ReportDepth(-1); + if (command.DeadLetter is { } letter) _deadLetters[command.MessageId] = letter; + if (command.RemovedDeadLetters is { } removed) + { + foreach (var key in removed) _deadLetters.Remove(key); + } + break; + case OutboxCommandKind.RemoveDeadLetters: + foreach (var key in command.RemovedDeadLetters!) _deadLetters.Remove(key); + break; + case OutboxCommandKind.Snapshot: + var snapshot = command.Snapshot ?? throw new InvalidOperationException("A snapshot command requires its state."); + var previousCount = _messages.Count; + _messages.Clear(); + _deadLetters.Clear(); + foreach (var restored in snapshot.Messages) _messages.Add(restored.Message.Envelope.MessageId, restored); + foreach (var restored in snapshot.DeadLetters) _deadLetters.Add(restored.Message.Envelope.MessageId, restored); + ReportDepth(_messages.Count - previousCount); + break; + default: + throw new NotSupportedException($"Outbox command '{command.Kind}' is not supported."); + } + } + + internal bool RemoveDeadLetter(HierarchicalKey id) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + if (!_deadLetters.ContainsKey(id)) return false; + Emit(new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [id] }); + return true; + } +} + +internal enum OutboxCommandKind +{ + Enqueue = 1, + Delivery = 2, + RemoveDeadLetters = 3, + Snapshot = 4 +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxCommand")] +internal sealed class OutboxCommand +{ + [Id(0)] public OutboxCommandKind Kind { get; init; } + [Id(1)] public OutboxMessage? Message { get; init; } + [Id(2)] public HierarchicalKey MessageId { get; init; } + [Id(3)] public OutboxMessageState? State { get; init; } + [Id(4)] public OutboxDeadLetter? DeadLetter { get; init; } + [Id(5)] public HierarchicalKey[]? RemovedDeadLetters { get; init; } + [Id(6)] public OutboxSnapshot? Snapshot { get; init; } +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxSnapshot")] +internal sealed class OutboxSnapshot +{ + [Id(0)] public OutboxEntry[] Messages { get; init; } = []; + [Id(1)] public OutboxDeadLetter[] DeadLetters { get; init; } = []; +} + +[GenerateSerializer, Alias("Orleans.Messaging.OutboxEntry")] +internal sealed class OutboxEntry +{ + [Id(0)] public required OutboxMessage Message { get; init; } + [Id(1)] public required OutboxMessageState State { get; set; } + [field: NonSerialized] + [JsonIgnore] public long Sequence { get; init; } +} + +[JsonSerializable(typeof(OutboxCommand))] +[JsonSerializable(typeof(OutboxSnapshot))] +internal sealed partial class OutboxJsonContext : JsonSerializerContext; diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs new file mode 100644 index 00000000000..23d34c210cc --- /dev/null +++ b/src/Orleans.Messaging/Outbox.cs @@ -0,0 +1,434 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Messaging.Configuration; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging; + +/// +/// Delivers journal-acknowledged messages to their receivers' inboxes. +/// +internal sealed partial class Outbox : IStateMachine, IDurableValueCommandHandler, IOutbox +{ + private readonly MessagingPump _pump; + private JournalStreamWriter _writer; + private readonly Dictionary _messages = []; + private readonly Dictionary _deadLetters = []; + private Task? _deliveries; + + public Outbox(MessagingPump pump) + { + _pump = pump; + pump.StateManager.RegisterStateMachine(MessagingStateNames.Outbox, this); + pump.Register(this); + } + + private MessagingShared Shared => _pump.Shared; + private DurableMessagingOptions Options => _pump.Shared.Options; + private ILogger Logger => _pump.Shared.OutboxLogger; + private TimeProvider TimeProvider => _pump.Shared.TimeProvider; + private GrainId GrainId => _pump.GrainContext.GrainId; + + internal IReadOnlyDictionary MessageState => _messages; + internal IReadOnlyDictionary DeadLetterState => _deadLetters; + internal int Count => _messages.Count; + + public void Send(OutboxMessage message) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + EnvelopeValidation.Validate(message); + if (_messages.TryGetValue(message.Envelope.MessageId, out var existing)) + { + if (!EnvelopeEquivalence.AreEquivalent(existing.Message, message)) + { + throw new InvalidOperationException( + $"The outbox already contains a different envelope with message ID '{message.Envelope.MessageId}'."); + } + return; + } + + try + { + Emit(new() { Kind = OutboxCommandKind.Enqueue, Message = message, State = new() { EnqueuedAt = TimeProvider.GetUtcNow() } }); + Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); + } + catch (Exception exception) + { + FailStaging(exception); + } + } + + private void OnRecoveryCompleted() + { + foreach (var (key, entry) in _messages) + { + var message = entry.Message; + EnvelopeValidation.Validate(message); + if (entry.State.EnqueuedAt is null) + { + throw new InvalidOperationException("A recovered outbox message requires its enqueue time."); + } + if (key != message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered outbox key does not match its command identity."); + } + } + foreach (var letter in _deadLetters.Values) EnvelopeValidation.Validate(letter.Message); + _capturedSequence = _durableSequence = _lastSequence; + } + + private void ResetState() + { + ReportDepth(-_messages.Count); + _messages.Clear(); + _deadLetters.Clear(); + _lastSequence = _capturedSequence = _durableSequence = 0; + } + + [DoesNotReturn] + private void FailStaging(Exception exception) + { + LogStagingFailed(Logger, exception); + _pump.Fail(exception, "Outbox staging failed."); + ExceptionDispatchInfo.Throw(exception); + } + + private async ValueTask PersistAsync() + { + try + { + await _pump.StateManager.WriteStateAsync(CancellationToken.None).ConfigureAwait(true); + } + catch (Exception exception) + { + _pump.Fail(exception, "Outbox persistence failed."); + throw; + } + } + + private List? SelectMessages() + { + var now = TimeProvider.GetUtcNow(); + List? candidates = null; + foreach (var entry in _messages.Values) + { + if (entry.Sequence <= _durableSequence && IsReadyForAttempt(entry.State, now)) + { + (candidates ??= []).Add(entry); + if (candidates.Count == Options.OutboxBatchSize) break; + } + } + if (candidates is null) + { + LogNoMessages(Logger, Count); + return null; + } + LogDeliveringMessages(Logger, candidates.Count); + return candidates; + } + + private async Task DeliverAsync(OutboxEntry entry, CancellationToken cancellationToken) + { + if (entry.State.EnqueuedAt is { } enqueuedAt + && MessagingTime.IsExpired(TimeProvider.GetUtcNow(), enqueuedAt, Options.MaxOutboxRetryAge)) + { + return new(entry, null, $"The message exceeded the maximum retry age of {Options.MaxOutboxRetryAge}.", TimeSpan.Zero, true); + } + var start = Stopwatch.GetTimestamp(); + var message = entry.Message; + try + { + var target = message.ReceiverId == GrainId + ? _pump.GrainContext.GetGrainExtension() + : Shared.GrainFactory.GetGrain(message.ReceiverId); + var result = await target.DeliverAsync(message.ToInboxMessage(), cancellationToken).ConfigureAwait(true); + return new(entry, result, null, Stopwatch.GetElapsedTime(start), false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return new(entry, null, "Delivery was canceled.", Stopwatch.GetElapsedTime(start), false); + } + catch (Exception exception) + { + if (Logger.IsEnabled(LogLevel.Error)) + { + LogDeliveryError(Logger, exception, message.Envelope.MessageId, message.ReceiverId); + } + return new(entry, null, exception.ToString(), Stopwatch.GetElapsedTime(start), false); + } + } + + /// Advances outbox delivery and returns whether delivery outcomes were applied. + internal async ValueTask AdvanceAsync(CancellationToken cancellationToken) + { + if (_deliveries is { } pending) + { + if (!pending.IsCompleted) return false; + _deliveries = null; + cancellationToken.ThrowIfCancellationRequested(); + await ApplyDeliveriesAsync(pending.Result).ConfigureAwait(true); + return true; + } + + var candidates = SelectMessages(); + if (candidates is null) return false; + var attempts = new Task[candidates.Count]; + for (var index = 0; index < attempts.Length; index++) + { + var entry = candidates[index]; + attempts[index] = DeliverAsync(entry, _pump.ShutdownToken); + if (entry.Message.ReceiverId == GrainId) + { + await attempts[index].ConfigureAwait(true); + } + } + + var deliveries = Task.WhenAll(attempts); + if (!deliveries.IsCompleted) + { + _deliveries = deliveries; + SignalOnCompletionAsync(deliveries).Ignore(); + return false; + } + + cancellationToken.ThrowIfCancellationRequested(); + await ApplyDeliveriesAsync(deliveries.Result).ConfigureAwait(true); + return true; + } + + private async Task SignalOnCompletionAsync(Task deliveries) + { + await deliveries.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + _pump.Signal(); + } + + private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) + { + Dictionary? deadLetters = null; + int applied = 0, delivered = 0, backpressured = 0; + try + { + foreach (var outcome in outcomes) + { + // Outcomes for entries which were replaced or removed since the attempt started are stale. + var id = outcome.Entry.Message.Envelope.MessageId; + if (!_messages.TryGetValue(id, out var current) || !ReferenceEquals(current, outcome.Entry)) continue; + + var status = outcome.Result?.Status; + if (status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.DeadLettered) + { + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id }); + delivered++; + } + else + { + if (status == DeliveryStatus.Backpressured) backpressured++; + var (retry, deadLetter) = PrepareRetry(outcome, ref deadLetters); + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id, State = retry, DeadLetter = deadLetter }); + } + RecordDeliveryMetrics(outcome); + applied++; + } + if (deadLetters is not null) ApplyDeadLetters(deadLetters); + } + catch (Exception exception) + { + FailStaging(exception); + } + if (applied == 0) return; + await PersistAsync().ConfigureAwait(true); + LogDeliveryComplete(Logger, delivered, backpressured, applied - delivered - backpressured, Count); + } + + private void ApplyDeadLetters(Dictionary entries) + { + foreach (var key in _deadLetters.Keys.Where(key => !entries.ContainsKey(key)).ToArray()) + { + Emit(new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [key] }); + } + foreach (var entry in entries) + { + if (!_deadLetters.TryGetValue(entry.Key, out var existing) || !ReferenceEquals(existing, entry.Value)) + { + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = entry.Key, DeadLetter = entry.Value }); + } + } + } + + private (OutboxMessageState? Retry, OutboxDeadLetter? DeadLetter) PrepareRetry( + DeliveryOutcome outcome, + ref Dictionary? deadLetters) + { + var previous = outcome.Entry.State; + var status = outcome.Result?.Status; + var now = TimeProvider.GetUtcNow(); + var attemptCount = outcome.Expired ? previous.AttemptCount : checked(previous.AttemptCount + 1); + var error = outcome.Error ?? status switch + { + DeliveryStatus.Backpressured => "The receiver is backpressured.", + DeliveryStatus.HandlerNotFound => outcome.Result?.Message ?? "The receiver has no registered inbox handler.", + _ => $"Unexpected delivery status '{status}'." + }; + if (outcome.Expired || attemptCount >= Options.MaxDeliveryAttempts + || MessagingTime.IsExpired(now, previous.EnqueuedAt!.Value, Options.MaxOutboxRetryAge)) + { + var letter = new OutboxDeadLetter + { + Message = outcome.Entry.Message, + DeadLetteredAt = now, + Reason = error, + AttemptCount = attemptCount + }; + deadLetters ??= _deadLetters.ToDictionary(static pair => pair.Key, static pair => pair.Value); + DeadLetterRetention.Compact(deadLetters, now, Options.DeadLetterRetentionPeriod, + Options.MaxRetainedDeadLetters, static entry => entry.DeadLetteredAt, + reservedCapacity: deadLetters.ContainsKey(letter.Message.Envelope.MessageId) ? 0 : 1); + deadLetters[letter.Message.Envelope.MessageId] = letter; + return (null, letter); + } + var exponent = Math.Min(attemptCount - 1, DurableMessagingOptions.MaximumBackoffExponent); + return (new OutboxMessageState + { + AttemptCount = attemptCount, + LastError = error, + EnqueuedAt = previous.EnqueuedAt, + NextAttemptAt = MessagingTime.AddClamped(now, TimeSpan.FromTicks(Options.BackpressureRetryDelay.Ticks * (1L << exponent))) + }, null); + } + + private void RecordDeliveryMetrics(DeliveryOutcome outcome) + { + var message = outcome.Entry.Message; + var status = outcome.Result?.Status; + var metricStatus = status switch + { + DeliveryStatus.Accepted => "accepted", + DeliveryStatus.Duplicate => "duplicate", + DeliveryStatus.DeadLettered => "deadlettered", + DeliveryStatus.Backpressured => "backpressured", + DeliveryStatus.HandlerNotFound => "handler_not_found", + null => "error", + _ => status.Value.ToString().ToLowerInvariant() + }; + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, metricStatus); + Shared.Instruments.OnOutboxDeliveryDuration(outcome.Duration, Shared.GrainType); + switch (status) + { + case DeliveryStatus.Accepted: + case DeliveryStatus.Duplicate: + case DeliveryStatus.DeadLettered: + if (Logger.IsEnabled(LogLevel.Debug)) + { + LogMessageDelivered(Logger, message.Envelope.MessageId, message.ReceiverId, status.Value); + } + break; + case DeliveryStatus.Backpressured: + LogDeliveryBackpressured(Logger, message.Envelope.MessageId, message.ReceiverId); + break; + case DeliveryStatus.HandlerNotFound: + if (Logger.IsEnabled(LogLevel.Warning)) + { + LogDeliveryHandlerNotFound(Logger, message.Envelope.MessageId, message.ReceiverId, outcome.Result?.Message); + } + break; + case { } unexpected: + if (Logger.IsEnabled(LogLevel.Warning)) + { + LogUnexpectedDeliveryStatus(Logger, unexpected, message.Envelope.MessageId, message.ReceiverId); + } + break; + } + } + + private bool IsReadyForAttempt(OutboxMessageState state, DateTimeOffset now) => + state.EnqueuedAt is { } enqueuedAt && MessagingTime.IsExpired(now, enqueuedAt, Options.MaxOutboxRetryAge) + || state.NextAttemptAt is null || state.NextAttemptAt <= now; + + private DateTimeOffset GetNextAttemptAt(OutboxMessageState state, DateTimeOffset now) + { + var retryAt = state.NextAttemptAt ?? now; + var expiresAt = state.EnqueuedAt is { } enqueuedAt ? MessagingTime.AddClamped(enqueuedAt, Options.MaxOutboxRetryAge) : retryAt; + return retryAt <= expiresAt ? retryAt : expiresAt; + } + + /// + /// Returns the delay until the next delivery attempt is due, or when the outbox is empty. + /// + internal TimeSpan? GetNextWakeupDelay() + { + if (Count == 0) return null; + var now = TimeProvider.GetUtcNow(); + var next = DateTimeOffset.MaxValue; + foreach (var entry in _messages.Values) + { + var at = GetNextAttemptAt(entry.State, now); + if (at < next) next = at; + } + return next > now ? next - now : TimeSpan.Zero; + } + + internal async Task StartAsync(CancellationToken cancellationToken) + { + if (_deadLetters.Count > Options.MaxRetainedDeadLetters || _deadLetters.Values.Any(entry => + MessagingTime.IsExpired(TimeProvider.GetUtcNow(), entry.DeadLetteredAt, Options.DeadLetterRetentionPeriod))) + { + var retained = _deadLetters.ToDictionary(static pair => pair.Key, static pair => pair.Value); + DeadLetterRetention.Compact(retained, TimeProvider.GetUtcNow(), + Options.DeadLetterRetentionPeriod, Options.MaxRetainedDeadLetters, static entry => entry.DeadLetteredAt); + try + { + ApplyDeadLetters(retained); + } + catch (Exception exception) + { + FailStaging(exception); + } + await PersistAsync().ConfigureAwait(true); + } + } + + /// Waits for in-flight deliveries, which observe the pump's shutdown token. + internal Task StopAsync() => (Task?)_deliveries ?? Task.CompletedTask; + + /// Releases the reported depth once the activation's messaging state is discarded. + internal void ReleaseDepth() => ReportDepth(-_messages.Count); + + private void ReportDepth(int delta) => Shared.Instruments.OnOutboxDepthChanged(Shared.GrainType, delta); + + [LoggerMessage(Level = LogLevel.Error, Message = "Outbox command staging failed.")] + private static partial void LogStagingFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Debug, Message = "No durable messages to deliver (all {Count} messages are still pending)")] + private static partial void LogNoMessages(ILogger logger, int count); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Delivering {Count} durable messages from outbox")] + private static partial void LogDeliveringMessages(ILogger logger, int count); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Delivered message {MessageId} to {ReceiverId} (Status: {Status})")] + private static partial void LogMessageDelivered(ILogger logger, HierarchicalKey messageId, GrainId receiverId, DeliveryStatus status); + + [LoggerMessage(Level = LogLevel.Warning, Message = "No inbox handler for message {MessageId} to {ReceiverId}: {Message}")] + private static partial void LogDeliveryHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId receiverId, string? message); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Backpressured delivering message {MessageId} to {ReceiverId}, will retry later")] + private static partial void LogDeliveryBackpressured(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Unexpected delivery status {Status} for message {MessageId} to {ReceiverId}")] + private static partial void LogUnexpectedDeliveryStatus(ILogger logger, DeliveryStatus status, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Error delivering message {MessageId} to {ReceiverId}")] + private static partial void LogDeliveryError(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + + [LoggerMessage(Level = LogLevel.Information, Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] + private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); + + private readonly record struct DeliveryOutcome(OutboxEntry Entry, DeliveryResult? Result, string? Error, TimeSpan Duration, bool Expired); +} diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 2c5382dc857..33ee10f5b8c 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -1,10 +1,10 @@ # Microsoft Orleans Messaging -`Microsoft.Orleans.Messaging` provides durable, grain-scoped inbox processing for -Microsoft Orleans, built on Orleans Journaling and Durable Jobs, together with the -messaging contracts for inboxes and outboxes. A grain's inbox accepts messages durably, -persists them with the grain's other journaled state, processes each at least once, -and deduplicates them by message ID. +`Microsoft.Orleans.Messaging` provides grain-scoped durable inboxes and outboxes +built on Orleans Journaling and Durable Jobs. A grain sends a message by staging it +in its outbox; the message persists with the grain's other journaled state and is +delivered to the receiver's inbox, which processes it at least once and deduplicates +it by message ID. ## Setup @@ -14,7 +14,7 @@ capacity, batch sizes, retries, deduplication, and dead-letter retention; option validated at startup. Register typed subjects with `AddMessageType(subject)`. Messaging grains derive from `DurableGrain` or implement `IMessagingGrain`, and -constructor-inject `IInbox`. During activation, the messaging grain-type +constructor-inject `IInbox` and `IOutbox`. During activation, the messaging grain-type configurator installs the grain's messaging extension. Stateless worker grains are rejected; reentrant and interleaving grains are supported, and the grain's code is responsible for correctness when calls interleave. @@ -72,9 +72,9 @@ A handler finishes each message with `IInboxHandlerContext.Complete()` or first failure, then moves to the dead letters. - Returning without either call, or calling both, deactivates the grain. -The inbox persists each batch's outcomes and business changes in one journal write. -Perform asynchronous and fallible preparation with local values, then apply state -changes and complete in one synchronous block. +The inbox persists each batch's outcomes, business changes, and outgoing messages in +one journal write. Perform asynchronous and fallible preparation with local values, +then apply state changes, send messages, and complete in one synchronous block. ## Send messages @@ -87,7 +87,8 @@ changes and complete in one synchronous block. carries in its request. `MessageType.Create` and `OutboxMessage.ToInboxMessage()` build messages for direct -inbox delivery. +inbox delivery. Sending equivalent content with a pending ID coalesces with the +original message; different content throws `InvalidOperationException`. ## Journal payloads once @@ -132,9 +133,9 @@ construction or activation setup, before journal recovery. ## Delivery and processing -`IMessagingExtension.DeliverAsync` delivers a message to a grain's inbox. Accepted inbox -messages are processed only after the receiver's journal write completes. Each -receiver returns a `DeliveryResult`: +Staged outgoing messages are delivered only after the sender's journal write +completes. Accepted inbox messages are processed only after the receiver's journal +write completes. Each receiver returns a `DeliveryResult`: - `Accepted`: the message is durable in the inbox, or, for a message an inbox sink accepted, in the sink's state together with its processed record. @@ -145,29 +146,35 @@ receiver returns a `DeliveryResult`: The inbox admits scheduled messages regardless of `MaxCapacity`. - `HandlerNotFound`: the receiver has no registered handlers. -Each activation runs one messaging pump on its scheduler, processing inbox batches. -One durable job per grain wakes the pump while the inbox or a registered work source -holds work. Journaled state determines which messages are due, so each wake-up -processes exactly the ready work. +The outbox removes accepted and duplicate messages, and retries other outcomes with +exponential backoff. It dead-letters a message after `MaxDeliveryAttempts` or once it +is older than `MaxOutboxRetryAge`. Delivery order is unspecified. + +Each activation runs one messaging pump on its scheduler, processing inbox batches +and delivering outbox batches. One durable job per grain wakes the pump while either +queue or a registered work source holds work. Journaled state determines which +messages are due, so each wake-up processes exactly the ready work. ## State -The inbox is an event-sourced journal state named `orleans.messaging.inbox`, and works -with the JSON and Orleans binary journal formats. It persists alongside the grain's -other journaled state. After the grain's state is deleted, it starts empty and -continues working. +The inbox and the outbox are event-sourced journal states named +`orleans.messaging.inbox` and `orleans.messaging.outbox`, and work with the JSON and +Orleans binary journal formats. They persist alongside the grain's other journaled +state. After the grain's state is deleted, both start empty and continue working. Completed message IDs are retained for `DeduplicationWindow`. Dead letters are -retained up to `DeadLetterRetentionPeriod` and `MaxRetainedDeadLetters`. +retained up to `DeadLetterRetentionPeriod` and `MaxRetainedDeadLetters` for each queue. ## Diagnostics -`IMessagingDiagnostics` exposes `InboxDeadLetters`, with each record's message, -reason, attempt count, and dead-letter time. `RemoveInboxDeadLetter` removes a record, -durably at the grain's next journal write. +`IMessagingDiagnostics` exposes `InboxDeadLetters` and `OutboxDeadLetters`, with each +record's message, reason, attempt count, and dead-letter time. +`RemoveInboxDeadLetter` and `RemoveOutboxDeadLetter` remove records, durably at the +grain's next journal write. -The `Microsoft.Orleans` meter publishes counters for received and processed messages -tagged by grain type and status, a histogram for processing duration, and the -`orleans-messaging-inbox-depth` up-down counter for pending work. A message an inbox +The `Microsoft.Orleans` meter publishes counters for received, processed, sent, and +delivered messages tagged by grain type and status; histograms for processing and +delivery duration; and `orleans-messaging-inbox-depth` and +`orleans-messaging-outbox-depth` up-down counters for pending work. A message an inbox sink accepts counts as received `accepted` and processed `success`; one which the sink dead-letters counts as received and processed `dead_lettered`. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 9f4795d5a6c..8bb13e751b7 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -206,7 +206,10 @@ public partial interface IMessagingDiagnostics { System.Collections.Generic.IReadOnlyList> InboxDeadLetters { get; } + System.Collections.Generic.IReadOnlyList> OutboxDeadLetters { get; } + bool RemoveInboxDeadLetter(MessageId messageId); + bool RemoveOutboxDeadLetter(MessageId messageId); } [Alias("IMessagingExtension")] diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index 6c76e3582a6..ff64e0ef3bb 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -168,6 +168,7 @@ public void PayloadUsesOrdinaryExternallyBoundOrleansSerializer() [InlineData("InboxMessage")] [InlineData("OutboxMessage")] [InlineData("InboxDeadLetter")] + [InlineData("OutboxDeadLetter")] public void MessageAndDeadLetterCopyPreservesMetadataAndIsolation(string kind) { using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 2d1bd13ff03..7510d9e00f8 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -12,8 +12,10 @@ namespace Orleans.Messaging.Tests.Contracts; public sealed class MessagingCommandSerializationTests { private static readonly DateTimeOffset Now = new(2040, 1, 2, 3, 4, 5, TimeSpan.Zero); + private static readonly GrainId Receiver = GrainId.Create("receiver", "1"); public static TheoryData InboxCommands => new(Enumerable.Range(0, CreateInboxCommands().Length)); + public static TheoryData OutboxCommands => new(Enumerable.Range(0, CreateOutboxCommands().Length)); [Theory] [MemberData(nameof(InboxCommands))] @@ -31,6 +33,22 @@ public void InboxCommand_RoundTripsThroughJsonAndOrleansSerialization(int index) Assert.Equal(json, JsonSerializer.Serialize(fromBinary!, InboxJsonContext.Default.InboxCommand)); } + [Theory] + [MemberData(nameof(OutboxCommands))] + public void OutboxCommand_RoundTripsThroughJsonAndOrleansSerialization(int index) + { + var command = CreateOutboxCommands()[index]; + var json = JsonSerializer.Serialize(command, OutboxJsonContext.Default.OutboxCommand); + var fromJson = JsonSerializer.Deserialize(json, OutboxJsonContext.Default.OutboxCommand)!; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(command)); + + Assert.Equal(command.Kind, fromJson.Kind); + Assert.Equal(json, JsonSerializer.Serialize(fromJson, OutboxJsonContext.Default.OutboxCommand)); + Assert.Equal(json, JsonSerializer.Serialize(fromBinary!, OutboxJsonContext.Default.OutboxCommand)); + } + [Fact] public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWhenUnset() { @@ -57,6 +75,8 @@ public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWh private static InboxMessage InboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt) }; + private static OutboxMessage OutboxMessage(string id) => new() { Envelope = Envelope(id), ReceiverId = Receiver }; + private static Envelope Envelope(string id, DateTimeOffset? deliverAt = null) => new() { MessageId = MessageId.Create("orders", id), @@ -94,4 +114,25 @@ private static InboxCommand[] CreateInboxCommands() } ]; } + + private static OutboxCommand[] CreateOutboxCommands() + { + var deadLetter = new OutboxDeadLetter { Message = OutboxMessage("dead"), DeadLetteredAt = Now, Reason = "rejected", AttemptCount = 3 }; + return + [ + new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() { EnqueuedAt = Now } }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", EnqueuedAt = Now } }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "dead"), DeadLetter = deadLetter }, + new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [HierarchicalKey.Create("orders", "dead")] }, + new() + { + Kind = OutboxCommandKind.Snapshot, + Snapshot = new() + { + Messages = [new() { Message = OutboxMessage("pending"), State = new() { AttemptCount = 1, EnqueuedAt = Now } }], + DeadLetters = [deadLetter] + } + } + ]; + } } diff --git a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs new file mode 100644 index 00000000000..28d4fc3932e --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs @@ -0,0 +1,176 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class JsonJournalMessagingTests(JsonJournalMessagingTests.Fixture fixture) : IClassFixture +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + private InProcessTestCluster Cluster => fixture.Cluster; + private JsonMessagingProbe Probe => fixture.Probe; + + [Fact] + public async Task JsonJournal_RecoversOutboxAndInboxStateAcrossActivations() + { + var sender = Cluster.Client.GetGrain(Guid.NewGuid()); + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + var messageId = HierarchicalKey.Create("json", Guid.NewGuid().ToString("N")); + + // The sender deactivates after committing the send, so delivery runs from the outbox recovered from the JSON journal. + await sender.SendAndDeactivateAsync(receiver.GetGrainId(), messageId, 42); + Assert.Equal(new[] { 42 }, await Probe.WaitAsync(receiver.GetGrainId(), 1, Token)); + Assert.True(Cluster.TryGetGrainContext(receiver.GetGrainId(), out var receiverContext)); + Assert.Contains("Json", receiverContext.GetComponent()!.InboxCodec.GetType().Name, StringComparison.Ordinal); + Assert.Same(receiverContext.ActivationServices.GetRequiredService(), receiverContext.ActivationServices.GetRequiredService()); + while (await sender.GetOutboxCountAsync() != 0) + { + await Task.Delay(10, Token); + } + + await receiver.DeactivateAsync(); + await receiverContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); + var redelivery = type.Create(messageId, receiver.GetGrainId(), 42).ToInboxMessage(); + + Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync(redelivery, Token)).Status); + Assert.Equal(new[] { 42 }, Probe.Get(receiver.GetGrainId())); + } + + [Fact] + public async Task CustomInboxHandlers_FirstHandlerWhichCanHandleReceivesMessage() + { + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + var message = new InboxMessage + { + Envelope = new Envelope { MessageId = HierarchicalKey.Create("custom", Guid.NewGuid().ToString("N")), Payload = [7], Subject = "custom" } + }; + + var result = await receiver.AsReference().DeliverAsync(message, Token); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + Assert.Equal(new[] { CustomHandlerGrain.FirstMatch }, await Probe.WaitAsync(receiver.GetGrainId(), 1, Token)); + } + + public sealed class Fixture : IAsyncLifetime + { + public JsonMessagingProbe Probe { get; } = new(); + public InProcessTestCluster Cluster { get; private set; } = null!; + + public async ValueTask InitializeAsync() + { + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureSilo((_, silo) => + { + silo.AddVolatileJournalStorage(); + silo.Services.Configure(options => options.JournalFormatKey = "json"); + silo.UseInMemoryDurableJobs(); + silo.AddMessaging(); + silo.Services.AddSingleton(Probe); + }); + Cluster = builder.Build(); + await Cluster.DeployAsync(TestContext.Current.CancellationToken); + } + + public async ValueTask DisposeAsync() => await Cluster.DisposeAsync(); + } +} + +public interface ICustomHandlerGrain : IGrainWithGuidKey; + +public sealed class CustomHandlerGrain : Grain, ICustomHandlerGrain, IMessagingGrain +{ + internal const int Declined = 1; + internal const int FirstMatch = 2; + internal const int SecondMatch = 3; + + public CustomHandlerGrain(IInbox inbox, IGrainContext context, JsonMessagingProbe probe) => + inbox.Register(new Handler(Declined, false, context, probe)) + .Register(new Handler(FirstMatch, true, context, probe)) + .Register(new Handler(SecondMatch, true, context, probe)); + + private sealed class Handler(int id, bool canHandle, IGrainContext grain, JsonMessagingProbe probe) : IInboxHandler + { + public bool CanHandle(IInboxHandlerContext context) => canHandle && context.Message.Envelope.Subject == "custom"; + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + probe.Record(grain.GrainId, id); + context.Complete(); + return ValueTask.CompletedTask; + } + } +} +public interface IJsonMessagingGrain : IGrainWithGuidKey +{ + Task SendAndDeactivateAsync(GrainId destination, HierarchicalKey messageId, int value); + Task GetOutboxCountAsync(); + Task DeactivateAsync(); +} + +public sealed class JsonMessagingProbe +{ + private readonly ConcurrentDictionary> _received = new(); + + public void Record(GrainId grainId, int value) => _received.GetOrAdd(grainId, static _ => new()).Enqueue(value); + + public int[] Get(GrainId grainId) => _received.TryGetValue(grainId, out var values) ? values.ToArray() : []; + + public async Task WaitAsync(GrainId grainId, int count, CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (Get(grainId).Length < count) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException($"No message was handled by '{grainId}'."); + await Task.Delay(10, cancellationToken); + } + + return Get(grainId); + } +} + +public sealed class JsonMessagingGrain : Grain, IJsonMessagingGrain, IMessagingGrain +{ + internal const string Subject = "json/value"; + private readonly IJournaledStateManager _manager; + private readonly IOutbox _outbox; + private readonly MessageType _type; + private readonly JsonMessagingProbe _probe; + + public JsonMessagingGrain(IJournaledStateManager manager, IInbox inbox, IOutbox outbox, Serializer serializer, JsonMessagingProbe probe) + { + _manager = manager; + _outbox = outbox; + _probe = probe; + _type = new MessageType(Subject, serializer); + inbox.Register(_type, this, static (value, grain, context) => + { + grain._probe.Record(grain.GetGrainId(), value); + context.Complete(); + }); + } + + public async Task SendAndDeactivateAsync(GrainId destination, HierarchicalKey messageId, int value) + { + _outbox.Send(_type, messageId, destination, value); + await _manager.WriteStateAsync(CancellationToken.None); + DeactivateOnIdle(); + } + + public Task GetOutboxCountAsync() => Task.FromResult(_outbox.Count()); + + public Task DeactivateAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingActivationBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingActivationBehaviorTests.cs new file mode 100644 index 00000000000..9796d093abe --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingActivationBehaviorTests.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingActivationBehaviorTests : MessagingBehaviorTestBase +{ + public MessagingActivationBehaviorTests() : base(receiverOnly: false) + { + } + + [Fact] + public async Task ApplicationJournaledStateNamesDoNotCollideWithMessagingState() + { + var grain = NewGrain(); + + var snapshot = await grain.GetSnapshotAsync(); + + Assert.Equal(0, snapshot.InboxCount); + Assert.Equal(0, snapshot.OutboxCount); + } + [Fact] + public async Task PublicHosting_BindsRealOutboxAndRegisteredMessagingStates() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(grain).ActivationServices; + var outbox = services.GetRequiredService(); + var inbox = services.GetRequiredService(); + Assert.Same(outbox, services.GetRequiredService()); + Assert.Same(inbox, services.GetRequiredService()); + var manager = services.GetRequiredService(); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Outbox, out var outboxState)); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Inbox, out var inboxState)); + Assert.Same(outbox, outboxState); + Assert.Same(inbox, inboxState); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs new file mode 100644 index 00000000000..5a42533a25f --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs @@ -0,0 +1,487 @@ +using System.Globalization; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Orleans.Concurrency; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public class MessagingGrainTypeConfiguratorTests : MessagingBehaviorTestBase +{ + public MessagingGrainTypeConfiguratorTests() : this(new BootstrapClusterFixture()) + { + } + + protected MessagingGrainTypeConfiguratorTests(BootstrapClusterFixture fixture) : base(fixture) + { + } + + private BootstrapDeliveryProbe Delivery => ((BootstrapClusterFixture)Fixture).Delivery; + private BootstrapProbe Probe => ((BootstrapClusterFixture)Fixture).Probe; + private static CancellationToken Cancellation => TestContext.Current.CancellationToken; + + [Theory] + [InlineData(typeof(PlainBootstrapGrain))] + [InlineData(typeof(ApplicationBootstrapGrain))] + [InlineData(typeof(InterfaceBootstrapGrain))] + [InlineData(typeof(GenericBootstrapGrain))] + [InlineData(typeof(DurableBootstrapGrain))] + [InlineData(typeof(MarkedDurableBootstrapGrain))] + public async Task SelectedComposition_InstallsInboxExtensionAndReplaysFreshScopedState(Type grainClass) + { + var grain = CreateGrain(grainClass); + Assert.Equal(0, await grain.GetValueAsync()); + var first = Assert.Single(Probe.Get(grain.GetGrainId())); + var state = first.Context.ActivationServices.GetRequiredService(); + AssertComposition(first, state, grainClass, expectedActivationValue: 0); + var setup = GetSetup(first.Context); + Assert.Single(setup.GetInvocationList()); + await grain.SetValueAsync(41); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + + using var handler = Fixture.HandlerProbe.Arm(grain.GetGrainId(), BootstrapState.Route); + var envelope = CreateEnvelope(grain); + Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + await handler.WaitUntilEnteredAsync(); + Assert.Equal(41, first.Value!.Value); + Assert.Equal(1, first.Inbox!.Count); + Assert.Equal(0, first.Outbox!.Count()); + Assert.Equal(0, GetProcessedCount(first)); + Assert.Equal(0, state.HandlerCalls); + handler.Release(); + await Fixture.WaitUntilAsync(grain, () => state.HandlerCalls == 1 && first.Inbox!.Count == 0); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(1, GetProcessedCount(first)); + var output = Assert.Single(first.Outbox!.Messages()); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId())); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(1, state.HandlerCalls); + + var other = CreateGrain(grainClass); + Assert.Equal(0, await other.GetValueAsync()); + var isolated = Assert.Single(Probe.Get(other.GetGrainId())); + Assert.NotSame(first.Manager, isolated.Manager); + Assert.NotSame(first.Inbox, isolated.Inbox); + Assert.NotSame(first.Outbox, isolated.Outbox); + Assert.Empty(isolated.Outbox!.Messages()); + await other.DeactivateAsync(); + await isolated.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + + await grain.DeactivateAsync(); + await first.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, first.Disposals); + Assert.Equal(1, first.GrainDisposals); + Assert.Equal(1, state.Disposals); + Assert.Equal(42, await grain.GetValueAsync()); + var observations = Probe.Get(grain.GetGrainId()); + Assert.Equal(2, observations.Length); + var recovered = observations[1]; + var recoveredState = recovered.Context.ActivationServices.GetRequiredService(); + AssertComposition(recovered, recoveredState, grainClass, expectedActivationValue: 42); + Assert.NotSame(first.Context, recovered.Context); + Assert.NotSame(first.Inbox, recovered.Inbox); + Assert.NotSame(first.Outbox, recovered.Outbox); + Assert.Same(setup, GetSetup(recovered.Context)); + Assert.Equal(output.Envelope.MessageId, Assert.Single(recovered.Outbox!.Messages()).Envelope.MessageId); + Assert.Equal(1, GetProcessedCount(recovered)); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(0, recoveredState.HandlerCalls); + Assert.Equal(2, Fixture.Storage.GetReadCount(journal)); + + var delivered = Delivery.WaitForOutputAsync(grain.GetGrainId()); + var drained = Delivery.WaitForDrainAsync(grain.GetGrainId()); + Delivery.Release(); + var receipt = await delivered; + await drained; + Assert.Equal(output.Envelope.MessageId, receipt.MessageId); + Assert.Equal(42, receipt.Value); + Assert.Empty(recovered.Outbox!.Messages()); + var sink = Fixture.Client.GetGrain("capture"); + Assert.Equal(1, await sink.GetMessageCountAsync()); + Assert.Equal(DeliveryStatus.Duplicate, (await sink.AsReference().DeliverAsync(output.ToInboxMessage(), Cancellation)).Status); + Assert.Equal(1, await sink.GetMessageCountAsync()); + } + + [Fact] + public async Task OrdinarySend_SchedulesWakeupBeforeCaptureAndDispatchesAfterAck() + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + using var scheduling = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); + var operation = grain.SendValueAsync(42); + await scheduling.WaitUntilEnteredAsync(); + Assert.Equal(42, observation.Value!.Value); + Assert.Equal(1, observation.Outbox!.Count()); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + var storage = Fixture.Storage.BlockWrite(journal); + var delivered = Delivery.WaitForOutputAsync(grain.GetGrainId()); + Delivery.Release(); + scheduling.Continue(); + await storage.WaitUntilEnteredAsync(); + Assert.Single(observation.Outbox!.Messages()); + Assert.False(delivered.IsCompleted); + Assert.Equal(1, Fixture.JobManagerProbe.GetSuccessCount(MessagingPump.JobName, grain.GetGrainId())); + storage.Release(); + await operation; + var receipt = await delivered; + Assert.Equal(42, receipt.Value); + Assert.Equal(42, await grain.GetValueAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OrdinarySend_SchedulingFailurePreservesPendingBusinessForExplicitRetry(bool ambiguous) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (ambiguous) Fixture.JobManagerProbe.FailAfterNext(MessagingPump.JobName); + else Fixture.JobManagerProbe.FailNext(MessagingPump.JobName); + var prerequisite = await Assert.ThrowsAsync(() => grain.SendValueAsync(42)); + Assert.IsType(prerequisite.InnerException); + Assert.Equal(42, await grain.GetValueAsync()); + Assert.Equal(1, observation.Outbox!.Count()); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(ambiguous ? 1 : 0, Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId()).Count); + await grain.PersistAsync(); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Equal(42, TestApplicationProtocol.Read(Sessions, Assert.Single(observation.Outbox!.Messages())).Body); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId())); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OrdinarySend_FreshReplayResolvesActualWriteOutcome(bool committed) + { + var grain = CreateGrain(typeof(PlainBootstrapGrain)); + await grain.SetValueAsync(11); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + if (committed) Fixture.Storage.FailAfterWrite(journal); + else Fixture.Storage.FailWrite(journal); + await Assert.ThrowsAsync(() => grain.SendValueAsync(42)); + Assert.Equal(writes + (committed ? 1 : 0), Fixture.Storage.GetSuccessfulWriteCount(journal)); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId())); + await grain.DeactivateAsync(); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(committed ? 42 : 11, await grain.GetValueAsync()); + var recovered = Probe.Get(grain.GetGrainId())[^1]; + Assert.NotSame(observation.Context, recovered.Context); + Assert.Equal(committed ? 1 : 0, recovered.Outbox!.Count()); + if (committed) + { + Assert.Equal(42, TestApplicationProtocol.Read(Sessions, Assert.Single(recovered.Outbox!.Messages())).Body); + } + else + { + Assert.Empty(recovered.Outbox!.Messages()); + } + } + + [Fact] + public async Task SelectedWithoutConstructorDependencies_MaterializesMessagingStateBeforeRead() + { + var grain = Control(); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + try + { + await read.WaitUntilEnteredAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Same(observation.ConstructedGrain, observation.Context.GrainInstance); + var manager = observation.Context.ActivationServices.GetRequiredService(); + Assert.Equal(2, BootstrapState.ReadMessagingStates(manager).Count()); + Assert.Equal(0, observation.Activations); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + + await activation; + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + } + + [Theory] + [InlineData(typeof(UnselectedBootstrapGrain), false)] + [InlineData(typeof(JournalOnlyBootstrapGrain), true)] + public async Task UnselectedComposition_LeavesMessagingUnresolved(Type grainClass, bool journaled) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + await grain.PingAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Equal(1, observation.Activations); + Assert.Null(observation.Inbox); + Assert.Null(observation.Outbox); + Assert.Null(GetSetupOrDefault(observation.Context)); + Assert.Null(observation.Context.GetComponent()); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(journaled ? 1 : 0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + if (journaled) + { + Assert.NotNull(observation.Manager); + Assert.True(observation.Manager.TryGetStateMachine("bootstrap-journal-only", out var state)); + Assert.Same(observation.Value, state); + Assert.Empty(BootstrapState.ReadMessagingStates(observation.Manager)); + } + else + { + Assert.Null(observation.Manager); + } + + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(ReentrantBootstrapGrain))] + [InlineData(typeof(MayInterleaveBootstrapGrain))] + [InlineData(typeof(AlwaysInterleaveBootstrapGrain))] + [InlineData(typeof(MetadataReentrantBootstrapGrain))] + [InlineData(typeof(MetadataMayInterleaveBootstrapGrain))] + public async Task ReentrantAndInterleavingModels_AreAcceptedAndBindTheInbox(Type grainClass) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + await grain.PingAsync(); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.Equal(1, observation.Activations); + var inbox = observation.Context.ActivationServices.GetRequiredService(); + Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); + + // No handler is registered by control grains, so the installed extension reports that explicitly. + var result = await grain.AsReference().DeliverAsync(CreateEnvelope(grain), Cancellation); + Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); + Assert.False(observation.Context.Deactivated.IsCompleted); + } + + [Fact] + public async Task StatelessWorker_FailsBeforeStorageAndCleansScope() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("one activation", exception.ToString(), StringComparison.Ordinal); + Assert.Contains(nameof(StatelessBootstrapGrain), exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.NotNull(observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataStatelessPlacementBootstrapGrain), "StatelessWorkerPlacement")] + [InlineData(typeof(AliasedStatelessPlacementBootstrapGrain), BootstrapClusterFixture.StatelessPlacementAlias)] + public async Task ResolvedStatelessPlacement_FailsBeforeStorageAndCleansScope(Type grainClass, string placementKey) + { + Assert.False(grainClass.IsDefined(typeof(StatelessWorkerAttribute), inherit: true)); + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + var placement = services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type); + Assert.Equal(new StatelessWorkerAttribute().PlacementStrategy.GetType(), placement.GetType()); + if (placementKey == BootstrapClusterFixture.StatelessPlacementAlias) + { + Assert.Same(services.GetRequiredKeyedService(placementKey), placement); + } + + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("one activation", exception.ToString(), StringComparison.Ordinal); + Assert.Contains(grainClass.Name, exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.Equal(0, Fixture.Storage.GetCreationCount(JournalId.FromGrainId(grain.GetGrainId()))); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Theory] + [InlineData(typeof(MetadataOrdinaryPlacementBootstrapGrain), nameof(RandomPlacement))] + [InlineData(typeof(AliasedOrdinaryPlacementBootstrapGrain), BootstrapClusterFixture.OrdinaryPlacementAlias)] + public async Task ResolvedOrdinaryPlacement_WithoutMessagingConstructorDependencies_InitializesNormally(Type grainClass, string placementKey) + { + var grain = Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + var services = Fixture.Cluster.Silos[0].ServiceProvider; + var properties = services.GetRequiredService().GetGrainProperties(grain.GetGrainId().Type); + Assert.Equal(placementKey, properties.Properties[WellKnownGrainTypeProperties.PlacementStrategy]); + Assert.IsType(services.GetRequiredService().GetPlacementStrategy(grain.GetGrainId().Type)); + + var journal = JournalId.FromGrainId(grain.GetGrainId()); + var read = Fixture.Storage.BlockRead(journal); + var activation = grain.PingAsync(); + BootstrapObservation observation; + try + { + await read.WaitUntilEnteredAsync(); + observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.IsType(grainClass, observation.ConstructedGrain); + Assert.Equal(0, observation.Activations); + var activationServices = observation.Context.ActivationServices; + var manager = activationServices.GetRequiredService(); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Inbox, out var inbox)); + Assert.Same(activationServices.GetRequiredService(), inbox); + Assert.True(manager.TryGetStateMachine(MessagingStateNames.Outbox, out var outbox)); + Assert.Same(activationServices.GetRequiredService(), outbox); + Assert.Single(GetSetup(observation.Context).GetInvocationList()); + } + finally + { + read.Release(); + } + + await activation; + Assert.Equal(1, observation.Activations); + Assert.Equal(1, Fixture.Storage.GetCreationCount(journal)); + Assert.Equal(1, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task SetupFailure_DisposesScopeAndPreservesOriginalError() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + Assert.IsType(observation.ExpectedFailure); + Assert.Contains(observation.ExpectedFailure.Message, exception.ToString(), StringComparison.Ordinal); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(0, observation.Activations); + Assert.Equal(1, observation.GrainDisposals); + Assert.Equal(1, observation.Disposals); + Assert.NotNull(observation.Inbox); + Assert.NotNull(observation.Outbox); + AssertNoStorageWork(grain.GetGrainId()); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public async Task UnselectedEndpointDependency_FailsExplicitly() + { + var grain = Control(); + var exception = await Assert.ThrowsAnyAsync(() => grain.PingAsync()); + Assert.Contains("Messaging is not configured for grain type", exception.ToString(), StringComparison.Ordinal); + var observation = Assert.Single(Probe.Get(grain.GetGrainId())); + await observation.Context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Cancellation); + Assert.Equal(1, observation.Disposals); + Assert.Null(observation.Inbox); + Assert.Null(GetSetupOrDefault(observation.Context)); + AssertNoScheduledJobs(grain.GetGrainId()); + } + + [Fact] + public void RepeatedRegistration_InstallsOneTypeConfigurator() + { + var services = new ServiceCollection(); + var silo = Substitute.For(); + silo.Services.Returns(services); + silo.AddJournaling(); + services.AddMessaging(); + services.AddMessaging(); + var descriptor = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IConfigureGrainTypeComponents) + && descriptor.ImplementationType == typeof(MessagingGrainTypeConfigurator)); + Assert.Equal(ServiceLifetime.Singleton, descriptor.Lifetime); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(MessagingPump)); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(Inbox)); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(Outbox)); + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IOutbox)); + Assert.DoesNotContain(services, static descriptor => descriptor.IsKeyedService && descriptor.ServiceType == typeof(IOutbox)); + var extension = Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IGrainExtension)); + Assert.Equal(typeof(Orleans.DurableJobs.DurableJob).Assembly, Assert.IsAssignableFrom(extension.ServiceKey).Assembly); + Assert.Empty(typeof(IMessagingGrain).GetInterfaces()); + Assert.Empty(typeof(IMessagingGrain).GetMethods()); + Assert.False(typeof(IAddressable).IsAssignableFrom(typeof(IMessagingGrain))); + } + + private static void AssertComposition(BootstrapObservation observation, BootstrapState state, Type grainClass, int expectedActivationValue) + { + Assert.Equal(grainClass, observation.ConstructedGrain!.GetType()); + Assert.False(observation.InstanceAvailableInConstructor); + Assert.Same(observation.ConstructedGrain, state.GrainAtActivation); + Assert.Equal(2, state.MessagingStateCountAtActivation); + Assert.Equal(1, observation.Activations); + Assert.Equal(expectedActivationValue, state.ActivationValue); + var services = observation.Context.ActivationServices; + Assert.Same(observation.Manager, services.GetRequiredService()); + var applicationManager = services.GetRequiredService(); + Assert.Same(observation.Manager, applicationManager); + Assert.Same(observation.Value, applicationManager.GetOrAddState>("bootstrap-value")); + var inbox = services.GetRequiredService(); + var outbox = services.GetRequiredService(); + Assert.Same(inbox, observation.Inbox); + Assert.Same(outbox, observation.Outbox); + Assert.True(observation.Manager!.TryGetStateMachine(MessagingStateNames.Inbox, out var inboxState)); + Assert.Same(inbox, inboxState); + Assert.True(observation.Manager.TryGetStateMachine(MessagingStateNames.Outbox, out var outboxState)); + Assert.Same(outbox, outboxState); + Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); + Assert.Same(state, services.GetRequiredService()); + } + + private IBootstrapTestGrain CreateGrain(Type grainClass) => grainClass == typeof(GenericBootstrapGrain) + ? Fixture.Client.GetGrain>(Guid.NewGuid()) + : Fixture.Client.GetGrain(Guid.NewGuid(), grainClass.FullName!); + + private IBootstrapControlGrain Control() => Fixture.Client.GetGrain(Guid.NewGuid(), typeof(T).FullName!); + + private int _commandSequence; + + private InboxMessage CreateEnvelope(IAddressable grain) => + OutboxTestMessages.CreateApplication(Fixture.Client.ServiceProvider, + HierarchicalKey.Create("test", grain.GetGrainId().ToString(), "command", (++_commandSequence).ToString(CultureInfo.InvariantCulture)), + GrainId.Create("bootstrap-sender", "external"), grain.GetGrainId(), BootstrapState.Route, 1).ToInboxMessage(); + + private static int GetProcessedCount(BootstrapObservation observation) => + InboxTestAccess.GetProcessedCount(Assert.IsType(observation.Inbox)); + + private static Delegate GetSetup(IGrainContext context) => Assert.IsAssignableFrom(GetSetupOrDefault(context)); + + private static object? GetSetupOrDefault(IGrainContext context) + { + var shared = context.GetType().GetField("_shared", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(context)!; + return shared.GetType().GetField("_activationSetup", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(shared); + } + + private void AssertNoStorageWork(GrainId grainId) + { + var journal = JournalId.FromGrainId(grainId); + Assert.Equal(0, Fixture.Storage.GetInitializationCount(journal)); + Assert.Equal(0, Fixture.Storage.GetReadCount(journal)); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + + private void AssertNoScheduledJobs(GrainId grainId) => + Assert.Equal(0, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, grainId)); +} diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs index e32182de271..9620ef70fb5 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingMetricCardinalityTests.cs @@ -129,6 +129,8 @@ public sealed class MessagingInstrumentTagTests [Theory] [InlineData("inbox-messages-received", "accepted")] [InlineData("inbox-messages-processed", "success")] + [InlineData("outbox-messages-sent", null)] + [InlineData("outbox-messages-delivered", "duplicate")] public void Counters_EmitExactBoundedTags(string instrumentName, string? status) { using var factory = new TestMeterFactory(); @@ -138,7 +140,9 @@ public void Counters_EmitExactBoundedTags(string instrumentName, string? status) Action emit = instrumentName switch { "inbox-messages-received" => () => instruments.OnInboxMessageReceived(grainType, status!), - _ => () => instruments.OnInboxMessageProcessed(grainType, status!), + "inbox-messages-processed" => () => instruments.OnInboxMessageProcessed(grainType, status!), + "outbox-messages-sent" => () => instruments.OnOutboxMessageSent(grainType), + _ => () => instruments.OnOutboxMessageDelivered(grainType, status!), }; emit(); var measurement = Assert.Single(probe.Read(instrumentName)); @@ -148,13 +152,16 @@ public void Counters_EmitExactBoundedTags(string instrumentName, string? status) [Theory] [InlineData("inbox-processing-duration")] + [InlineData("outbox-delivery-duration")] public void Durations_RecordMillisecondsAndClampNegativeValues(string instrumentName) { using var factory = new TestMeterFactory(); var instruments = new Orleans.Messaging.MessagingInstruments(new OrleansInstruments(factory)); using var probe = new MessageMetricListener(factory.Meter); const string grainType = "instrument-test"; - Action record = duration => instruments.OnInboxProcessingDuration(duration, grainType); + Action record = instrumentName == "inbox-processing-duration" + ? duration => instruments.OnInboxProcessingDuration(duration, grainType) + : duration => instruments.OnOutboxDeliveryDuration(duration, grainType); record(TimeSpan.FromMilliseconds(12.5)); record(TimeSpan.FromMilliseconds(-10)); var values = probe.Read(instrumentName); @@ -177,9 +184,12 @@ public void DepthInstruments_RecordNonZeroDeltasTaggedByGrainType() instruments.OnInboxDepthChanged("instrument-test", 3); instruments.OnInboxDepthChanged("instrument-test", 0); instruments.OnInboxDepthChanged("instrument-test", -1); + instruments.OnOutboxDepthChanged("instrument-test", 4); var inboxDepth = probe.Read("inbox-depth"); + var outboxDepth = Assert.Single(probe.Read("outbox-depth")); Assert.Equal([3d, -1d], inboxDepth.Select(static row => row.Value)); - Assert.All(inboxDepth, row => + Assert.Equal(4, outboxDepth.Value); + Assert.All(inboxDepth.Append(outboxDepth), row => { Assert.Null(row.Unit); Assert.Equal(new[] { new KeyValuePair("grain_type", "instrument-test") }, row.Tags); diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs new file mode 100644 index 00000000000..ec8570a36ee --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs @@ -0,0 +1,87 @@ +using System.Collections.Immutable; +using System.Net; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.DurableJobs; +using Orleans.Hosting; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; +using Orleans.TestingHost; +using Orleans.Timers; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MessagingProviderCutoverTests +{ + private const string JournalFormatKey = "orleans-binary"; + private static CancellationToken Token => TestContext.Current.CancellationToken; + + [Fact] + public async Task PublicHosting_NamedFactories_CommitsCompositeEndpointsAndRecoversFreshActivation() + { + var state = new ControlledJournalStorageProvider(); + var snapshots = new NamedFactoryMessagingProbe(); + var builder = new InProcessTestClusterBuilder(1); + builder.ConfigureSilo((_, silo) => + { + silo.AddVolatileJournalStorage(); + silo.AddJournalStorage("state", services => + { + state.Configure(services.GetRequiredService>()); + return state; + }); + silo.AddVolatileJournalStorage("jobs"); + silo.UseJournaledDurableJobs(options => options.ActiveProviderName = "jobs"); + silo.Services.Configure(options => options.JournalFormatKey = JournalFormatKey); + silo.AddMessaging(); + silo.Services.AddSingleton(snapshots); + silo.Services.AddScoped(services => + { + var context = services.GetRequiredService(); + var manager = services.GetRequiredKeyedService("state") + .CreateStandalone(JournalId.FromGrainId(context.GrainId)); + ((ILifecycleParticipant)manager).Participate(context.ObservableLifecycle); + return manager; + }); + }); + await using var cluster = builder.Build(); + await cluster.DeployAsync(Token); + var sender = cluster.Client.GetGrain(Guid.NewGuid()); + var receiver = cluster.Client.GetGrain(Guid.NewGuid()); + var sink = cluster.Client.GetGrain(Guid.NewGuid()); + var logicalId = HierarchicalKey.Create("cutover", Guid.NewGuid().ToString("N")); + var receivedTask = snapshots.WaitAsync(receiver.GetGrainId(), Token); + var forwardedTask = snapshots.WaitAsync(sink.GetGrainId(), Token); + await sender.SendAsync(receiver.GetGrainId(), "messages/record", new TestMessage(logicalId, 1, "composite", sink.GetGrainId())); + var received = await receivedTask; + var forwarded = await forwardedTask; + Assert.Equal(0, received.OutboxCount); + Assert.Equal(0, forwarded.OutboxCount); + Assert.Equal(new MessageEffect(logicalId, 1, 1, "composite"), Assert.Single(received.Effects)); + Assert.Equal(Assert.Single(received.Effects), Assert.Single(forwarded.Effects)); + Assert.True(cluster.TryGetGrainContext(receiver.GetGrainId(), out var oldContext)); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + Assert.False(oldManager is IDurableStateManager); + await receiver.RequestDeactivationAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(received.ActivationId, recovered.ActivationId); + Assert.Equal(received.Effects, recovered.Effects); + Assert.True(cluster.TryGetGrainContext(receiver.GetGrainId(), out var newContext)); + Assert.NotSame(oldManager, newContext.ActivationServices.GetRequiredService()); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + Assert.NotNull(await state.CreateStorage(journal).GetMetadataAsync(Token)); + Assert.Null(await cluster.Silos[0].ServiceProvider.GetRequiredService().CreateStorage(journal).GetMetadataAsync(Token)); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs new file mode 100644 index 00000000000..20b5d057fdc --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs @@ -0,0 +1,64 @@ +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class MultiSiloMessagingCollection : ICollectionFixture +{ + public const string Name = "Messaging multi-silo cluster"; +} + +[Collection(MultiSiloMessagingCollection.Name)] +[TestSuite("Functional")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class MultiSiloMessagingFailoverTests(MultiSiloMessagingClusterFixture fixture) +{ + [Fact] + public async Task ReceiverSiloStops_DuringBlockedHandler_NewActivationRecoversFromJournalAndProcessesOnce() + { + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + using var barrier = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failover"); + var sender = fixture.Client.GetGrain(Guid.NewGuid()); + var logicalId = HierarchicalKey.Create("failover", Guid.NewGuid().ToString("N")); + fixture.JobManagerProbe.DuplicateNext(MessagingPump.JobName); + + await sender.SendAsync( + receiver.GetGrainId(), + "messages/failover", + new TestMessage(logicalId, 81, "failover")); + await barrier.WaitUntilEnteredAsync(); + Assert.True(fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId()).Count >= 2); + var before = fixture.GetSnapshot(receiver); + var owner = fixture.Cluster.Silos.Single( + silo => silo.SiloAddress.ToParsableString() == before.SiloAddress); + + await fixture.Cluster.KillSiloAsync(owner, TestContext.Current.CancellationToken); + await fixture.Cluster.WaitForLivenessToStabilizeAsync(); + var activationRequest = receiver.GetSnapshotAsync(); + barrier.Release(); + var reactivated = await activationRequest; + Assert.NotEqual(before.ActivationId, reactivated.ActivationId); + EndpointSnapshot recovered; + try + { + recovered = await fixture.WaitForEffectCountAsync(receiver, 1); + } + catch (TimeoutException exception) + { + var snapshot = await receiver.GetSnapshotAsync(); + throw new TimeoutException( + $"Recovery did not complete. Activation={snapshot.ActivationId}, silo={snapshot.SiloAddress}, inbox={snapshot.InboxCount}, effects={snapshot.Effects.Count}, deadLetters={snapshot.InboxDeadLetters.Count}.", + exception); + } + + Assert.Equal(reactivated.ActivationId, recovered.ActivationId); + Assert.NotEqual(before.SiloAddress, recovered.SiloAddress); + var effect = Assert.Single(recovered.Effects); + Assert.Equal(logicalId, effect.LogicalId); + Assert.Equal(1, effect.Count); + Assert.Equal(0, recovered.InboxCount); + Assert.Single(fixture.Cluster.Silos); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs new file mode 100644 index 00000000000..3cec0f71534 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs @@ -0,0 +1,241 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class OutboxDeliveryBehaviorTests : MessagingBehaviorTestBase +{ + public OutboxDeliveryBehaviorTests() : base(receiverOnly: false) + { + } + + [Fact] + public async Task StagedOutboxWithoutCommit_IsRemovedOnReactivationAndNeverDispatched() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var message = NewMessage(51, "uncommitted"); + + await sender.StageWithoutCommitAsync(receiver.GetGrainId(), "messages/uncommitted", message); + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + await sender.RequestDeactivationAsync(); + var reactivated = await sender.GetSnapshotAsync(); + + Assert.Equal(0, reactivated.OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task DuplicateOutboxEnqueue_UsesOneDurableWakeup() + { + var sender = NewGrain(); + var receiver = NewGrain(); + + await sender.SendDuplicateAsync( + receiver.GetGrainId(), + "messages/duplicate-outbox-enqueue", + NewMessage(54, "duplicate-enqueue")); + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal( + 1, + Fixture.JobManagerProbe.GetSuccessCount( + MessagingPump.JobName, + sender.GetGrainId())); + } + + [Fact] + public async Task CommittedOutbox_ToSameGrain_DeliversLocally() + { + var grain = NewGrain(); + + await grain.SendAsync( + grain.GetGrainId(), + "messages/loopback", + NewMessage(82, "loopback")); + var delivered = await Fixture.WaitForEffectCountAsync(grain, 1); + delivered = await Fixture.WaitForOutboxCountAsync(grain, 0); + + Assert.Equal("loopback", Assert.Single(delivered.Effects).Value); + Assert.Equal(0, delivered.InboxCount); + Assert.Empty(delivered.InboxDeadLetters); + Assert.Empty(delivered.OutboxDeadLetters); + } + + [Fact] + public async Task ReciprocalOutboxPumps_DoNotBlockInboxDelivery() + { + var first = NewGrain(); + var second = NewGrain(); + _ = await first.GetSnapshotAsync(); + _ = await second.GetSnapshotAsync(); + + var firstSend = first.SendAsync( + second.GetGrainId(), + "messages/reciprocal", + NewMessage(83, "first-to-second")); + var secondSend = second.SendAsync( + first.GetGrainId(), + "messages/reciprocal", + NewMessage(84, "second-to-first")); + await Task.WhenAll(firstSend, secondSend); + + var completed = await Task.WhenAll( + Fixture.SnapshotProbe.WaitAsync( + first.GetGrainId(), + static snapshot => snapshot.Effects.Any(effect => effect.Value == "second-to-first"), + TimeSpan.FromSeconds(10)), + Fixture.SnapshotProbe.WaitAsync( + second.GetGrainId(), + static snapshot => snapshot.Effects.Any(effect => effect.Value == "first-to-second"), + TimeSpan.FromSeconds(10))); + await Task.WhenAll( + Fixture.WaitForOutboxCountAsync(first, 0), + Fixture.WaitForOutboxCountAsync(second, 0)); + + Assert.Contains(completed[0].Effects, effect => effect.Value == "second-to-first"); + Assert.Contains(completed[1].Effects, effect => effect.Value == "first-to-second"); + } + + [Fact] + public async Task OutboxSchedulingPrerequisiteFailure_KeepsActivationHealthyAndRetryEstablishesWakeup() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + var message = NewMessage(55, "schedule-retry"); + Fixture.JobManagerProbe.FailAfterNext(MessagingPump.JobName); + + var prerequisite = await Assert.ThrowsAsync(() => sender.SendAsync(receiver.GetGrainId(), "messages/schedule-retry", message)); + Assert.IsType(prerequisite.InnerException); + Assert.False(oldContext.Deactivated.IsCompleted); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + var healthy = await sender.GetSnapshotAsync(); + Assert.Equal(before.ActivationId, healthy.ActivationId); + Assert.Same(oldManager, Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService()); + Assert.Equal(1, healthy.OutboxCount); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + try + { + await sender.RetryWriteStateAsync(); + await receiverWrite.WaitUntilEnteredAsync(); + var committed = Fixture.GetSnapshot(sender); + var jobs = Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId()); + Assert.NotEmpty(jobs); + Assert.All(jobs, job => + { + Assert.Null(job.Metadata); + Assert.Equal(sender.GetGrainId(), job.TargetGrainId); + }); + Assert.Equal(1, committed.OutboxCount); + } + finally + { + receiverWrite.Release(); + } + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("schedule-retry", Assert.Single(delivered.Effects).Value); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.True(Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId()) >= 1); + } + + [Fact] + public async Task FailedAtomicWrite_FencesOldObjectsAndFreshReplayOmitsEffectAndOutgoingMessage() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + await sender.StageEffectAsync(new MessageEffect(HierarchicalKey.Create("outbox", "failed-write"), 1, 53, "failed-write")); + Fixture.Storage.FailWrite(JournalId.FromGrainId(sender.GetGrainId())); + + var failure = await Assert.ThrowsAsync(() => sender.SendAsync( + receiver.GetGrainId(), "messages/write-failure", NewMessage(53, "failed-write"))); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(failure.Message, Assert.IsType( + await oldGrain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)).Message); + await Assert.ThrowsAsync(() => oldManager.WriteStateAsync(CancellationToken.None).AsTask()); + var recovered = await sender.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.NotSame(oldManager, Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService()); + Assert.Equal(0, recovered.OutboxCount); + Assert.Empty(recovered.Effects); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + } + + [Fact] + public async Task StopDeleteAndDeactivate_DiscardsUncommittedOutboxAndFreshOwnerCanSend() + { + var sender = NewGrain(); + var receiver = NewGrain(); + await sender.StageWithoutCommitAsync( + receiver.GetGrainId(), + "messages/delete-uncommitted", + NewMessage(75, "deleted-intent")); + + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + Assert.Equal(1, before.OutboxCount); + await sender.DeleteStateAndDeactivateAsync(); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var fresh = await sender.GetSnapshotAsync(); + + Assert.NotEqual(before.ActivationId, fresh.ActivationId); + Assert.Equal(0, fresh.OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + + var effect = new MessageEffect(HierarchicalKey.Create("outbox", "fresh-owner"), 1, 76, "fresh-owner"); + await sender.StageEffectAsync(effect); + var drained = Fixture.SnapshotProbe.WaitAsync( + sender.GetGrainId(), + snapshot => snapshot.ActivationId == fresh.ActivationId + && snapshot.OutboxCount == 0 + && snapshot.Effects.Contains(effect)); + await sender.SendAsync(receiver.GetGrainId(), "messages/after-delete", NewMessage(76, "after-delete")); + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + await drained; + Assert.Equal(0, (await sender.GetSnapshotAsync()).OutboxCount); + Assert.Equal("after-delete", Assert.Single(delivered.Effects).Value); + } + + [Fact] + public async Task BlockedInboxHandler_DoesNotStopIndependentOutboxAndInboxPumps() + { + var blocked = NewGrain(); + var independentSender = NewGrain(); + var independentReceiver = NewGrain(); + using var barrier = Fixture.HandlerProbe.Arm(blocked.GetGrainId(), "messages/blocked-pump"); + var blockedEnvelope = CreateEnvelope(blocked, NewMessage(61, "blocked"), "messages/blocked-pump"); + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(blocked, blockedEnvelope)).Status); + await barrier.WaitUntilEnteredAsync(); + await independentSender.SendAsync( + independentReceiver.GetGrainId(), + "messages/independent", + NewMessage(62, "independent")); + var independent = await Fixture.WaitForEffectCountAsync(independentReceiver, 1); + + Assert.Equal("independent", Assert.Single(independent.Effects).Value); + Assert.Empty(Fixture.GetSnapshot(blocked).Effects); + barrier.Release(); + Assert.Equal("blocked", Assert.Single((await Fixture.WaitForEffectCountAsync(blocked, 1)).Effects).Value); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxJournalRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxJournalRecoveryTests.cs new file mode 100644 index 00000000000..fe791cb72de --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/OutboxJournalRecoveryTests.cs @@ -0,0 +1,149 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class OutboxJournalRecoveryTests : MessagingBehaviorTestBase +{ + public OutboxJournalRecoveryTests() : base(receiverOnly: false) + { + } + + [Fact] + public async Task CommittedOutbox_DeactivationBeforeLocalFollowUp_RecoversPendingWork() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await sender.SendAndDeactivateAsync( + receiver.GetGrainId(), + "messages/outbox-crash-window", + NewMessage(52, "durable-wakeup")); + await receiverWrite.WaitUntilEnteredAsync(); + + Assert.Equal(1, Fixture.GetSnapshot(sender).OutboxCount); + Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + + receiverWrite.Release(); + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + var recovered = await Fixture.SnapshotProbe.WaitAsync( + sender.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.OutboxCount == 0); + + Assert.Equal("durable-wakeup", Assert.Single(delivered.Effects).Value); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + } + + [Fact] + public async Task OutboxRemovalWriteFailure_FencesOldManagerAndFreshActivationCleansUp() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var oldManager = oldContext.ActivationServices.GetRequiredService(); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + Fixture.Storage.FailWrite(JournalId.FromGrainId(sender.GetGrainId()), matchingWrite: 2); + + await sender.SendAsync(receiver.GetGrainId(), "messages/outbox-clear-retry", NewMessage(56, "outbox-clear-retry")); + _ = await Fixture.WaitForEffectCountAsync(receiver, 1); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.IsType(await oldGrain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + _ = await sender.GetSnapshotAsync(); + var cleaned = await Fixture.SnapshotProbe.WaitAsync(sender.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.OutboxCount == 0); + + Assert.NotSame(oldManager, Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService()); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + await sender.RequestDeactivationAsync(); + Assert.Equal(0, (await sender.GetSnapshotAsync()).OutboxCount); + } + + [Fact] + public async Task Outbox_SpuriousWakeupPreservesBlockedPendingWork() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + + await sender.SendAsync(receiver.GetGrainId(), "messages/spurious-outbox-wakeup", NewMessage(61, "spurious-wakeup")); + await receiverWrite.WaitUntilEnteredAsync(); + Assert.Equal(1, Fixture.GetSnapshot(sender).OutboxCount); + var attemptBaseline = Fixture.Metrics.GetCount("orleans-durablejobs-job-attempts-started"); + var manager = Fixture.Cluster.Silos[0].ServiceProvider.GetRequiredService(); + + try + { + await manager.ScheduleJobAsync( + new ScheduleJobRequest + { + Target = sender.GetGrainId(), + JobName = MessagingPump.JobName, + DueTime = DateTimeOffset.UtcNow + }, + TestContext.Current.CancellationToken); + await Fixture.Metrics.WaitForCountAsync("orleans-durablejobs-job-attempts-started", attemptBaseline + 1); + + Assert.Equal(1, Fixture.GetSnapshot(sender).OutboxCount); + } + finally + { + receiverWrite.Release(); + } + + Assert.Equal("spurious-wakeup", Assert.Single((await Fixture.WaitForEffectCountAsync(receiver, 1)).Effects).Value); + Assert.Equal(0, (await Fixture.WaitForOutboxCountAsync(sender, 0)).OutboxCount); + } + + [Fact] + public async Task Outbox_AmbiguousCommit_FreshReplayRestoresMessagesBeforeDelivery() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + _ = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(sender); + var journal = JournalId.FromGrainId(sender.GetGrainId()); + var recoveryRead = Fixture.Storage.BlockRead(journal); + var receiverWrite = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + Fixture.Storage.FailAfterWrite(journal); + + await Assert.ThrowsAsync(() => sender.SendAsync( + receiver.GetGrainId(), "messages/recovery-visibility", NewMessage(62, "recovery-visibility"))); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var activation = sender.GetSnapshotAsync(); + try + { + await recoveryRead.WaitUntilEnteredAsync(); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + recoveryRead.Release(); + await receiverWrite.WaitUntilEnteredAsync(); + var recovered = await activation; + + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal(1, recovered.OutboxCount); + } + finally + { + recoveryRead.Release(); + receiverWrite.Release(); + } + + var delivered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("recovery-visibility", Assert.Single(delivered.Effects).Value); + Assert.Equal(1, Assert.Single(delivered.Effects).Count); + Assert.Equal(0, (await Fixture.WaitForOutboxCountAsync(sender, 0)).OutboxCount); + } +} diff --git a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs new file mode 100644 index 00000000000..48a5c193076 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs @@ -0,0 +1,314 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicInboxHandlerTransactionTests : MessagingBehaviorTestBase +{ + public PublicInboxHandlerTransactionTests() : base(receiverOnly: false) + { + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task ArbitraryRawPayload_AcceptanceReplayForwardAndBusinessCompletionRequireNoApplicationCodec(int sends) + { + var receiver = Fixture.Client.GetGrain(Guid.NewGuid()); + var sink = Fixture.Client.GetGrain(Guid.NewGuid()); + await receiver.ConfigureForwardAsync(sink.GetGrainId(), sends); + _ = await sink.GetSnapshotAsync(); + Assert.True(Fixture.Cluster.TryGetGrainContext(receiver.GetGrainId(), out var original)); + Assert.True(Fixture.Cluster.TryGetGrainContext(sink.GetGrainId(), out var sinkContext)); + var sinkGrain = Assert.IsType(sinkContext.GrainInstance); + sinkGrain.ExpectedAcknowledgedEffects = sends; + var bytes = System.Text.Encoding.UTF8.GetBytes("raw application bytes\0\u03c0/forward").Concat(new byte[] { 0xff, 0x80, 0x01 }).ToArray(); + var envelope = new Envelope + { + MessageId = HierarchicalKey.Create("raw", Guid.NewGuid().ToString("N")), + Payload = bytes, + Headers = new Dictionary(StringComparer.Ordinal) { ["attachment"] = [0xff, 0x80, 0x01] } + }; + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync( + new InboxMessage { Envelope = envelope })).Status); + await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var accepted = Assert.IsType(Assert.IsType(original.GrainInstance).AcceptedSnapshot); + Assert.Equal(1, accepted.InboxCount); + Assert.Empty(accepted.Effects); + + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + using var replay = Fixture.Storage.BlockRead(journal); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), RawPayloadTestGrain.HandlerBarrier); + var activation = receiver.GetSnapshotAsync(); + await replay.WaitUntilEnteredAsync(); + replay.Release(); + await handler.WaitUntilEnteredAsync(); + Assert.True(Fixture.Cluster.TryGetGrainContext(receiver.GetGrainId(), out var current)); + Assert.NotSame(original, current); + var grain = Assert.IsType(current.GrainInstance); + var pending = Assert.Single(current.ActivationServices.GetRequiredService().Messages()); + Assert.Equal(bytes, pending.Envelope.Payload.ToArray()); + Assert.Null(pending.Envelope.Subject); + Assert.False(Assert.IsType>(pending.Envelope.Headers).ContainsKey("sender")); + Assert.Empty(grain.GetSnapshotForTest().Effects); + using var completion = Fixture.Storage.BlockWrite(journal); + handler.Release(); + await completion.WaitUntilEnteredAsync(); + var staged = grain.GetSnapshotForTest(); + Assert.Equal(0, staged.InboxCount); + Assert.Equal(sends, staged.OutboxCount); + Assert.Equal(1, staged.ProcessedCount); + Assert.Equal(new RawPayloadEffect(envelope.MessageId, Convert.ToBase64String(bytes), 1), Assert.Single(staged.Effects)); + var forwarded = current.ActivationServices.GetRequiredService().Messages().ToArray(); + Assert.Equal(sends, forwarded.Length); + Assert.Equal(sends, forwarded.Select(output => output.Envelope.MessageId).Distinct().Count()); + Assert.Contains(forwarded, output => output.Envelope.MessageId == envelope.MessageId); + Assert.All(forwarded, output => + { + Assert.Equal(sink.GetGrainId(), output.ReceiverId); + Assert.Equal(bytes, output.Envelope.Payload.ToArray()); + var headers = Assert.IsType>(output.Envelope.Headers); + Assert.Equal(new byte[] { 0xff, 0x80, 0x01 }, headers["attachment"]); + Assert.False(headers.ContainsKey("sender")); + Assert.Null(output.Envelope.Subject); + if (output.Envelope.MessageId == envelope.MessageId) + { + Assert.Same(pending.Envelope.Payload, output.Envelope.Payload); + Assert.Same(pending.Envelope.Headers, output.Envelope.Headers); + } + }); + Assert.False(grain.Acknowledged.IsCompleted); + Assert.False(sinkGrain.Acknowledged.IsCompleted); + completion.Release(); + await activation; + var delivered = await sinkGrain.Acknowledged.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var acknowledged = await grain.Acknowledged.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Equal(Assert.Single(acknowledged.Effects), Assert.Single(delivered.Effects, effect => effect.MessageId == envelope.MessageId)); + Assert.Equal(sends, delivered.Effects.Length); + Assert.All(delivered.Effects, effect => + { + Assert.Equal(Convert.ToBase64String(bytes), effect.Bytes); + Assert.Equal(1, effect.Count); + }); + Assert.Equal(0, acknowledged.OutboxCount); + Assert.Equal(sends, delivered.ProcessedCount); + Assert.Empty(current.ActivationServices.GetRequiredService().InboxDeadLetters); + Assert.Empty(current.ActivationServices.GetRequiredService().OutboxDeadLetters); + await receiver.RequestDeactivationAsync(); + await current.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await receiver.GetSnapshotAsync(); + Assert.NotEqual(acknowledged.ActivationId, recovered.ActivationId); + Assert.Equal(acknowledged.Effects, recovered.Effects); + Assert.Equal(1, recovered.ProcessedCount); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync( + new InboxMessage { Envelope = envelope }, TestContext.Current.CancellationToken)).Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + var sinkState = await sink.GetSnapshotAsync(); + Assert.Equal(sends, sinkState.Effects.Length); + Assert.All(sinkState.Effects, effect => Assert.Equal(1, effect.Count)); + } + + [Fact] + public async Task CaptureProbeSnapshot_RemainsStableAcrossLaterWrites() + { + var receiver = NewGrain(); + var first = new MessageEffect(HierarchicalKey.Create("capture", "first"), 1, 1, "first"); + var second = new MessageEffect(HierarchicalKey.Create("capture", "second"), 1, 2, "second"); + await receiver.StageEffectAsync(first); + await receiver.RetryWriteStateAsync(); + var grain = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); + var captures = grain.Captures; + var originalCount = captures.Count; + using var enumerator = captures.GetEnumerator(); + Assert.True(enumerator.MoveNext()); + var observed = new List { enumerator.Current }; + + await receiver.StageEffectAsync(second); + await receiver.RetryWriteStateAsync(); + + while (enumerator.MoveNext()) + { + observed.Add(enumerator.Current); + } + + Assert.Equal(originalCount, captures.Count); + Assert.Equal(originalCount, observed.Count); + Assert.Equal(captures, observed); + Assert.DoesNotContain(captures, capture => capture.Effects.Contains(second)); + Assert.Contains(grain.Captures, capture => capture.Effects.Contains(second)); + } + + [Fact] + public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndOutgoingAtomically() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var logicalId = HierarchicalKey.Create("atomic", Guid.NewGuid().ToString("N")); + var envelope = CreateEnvelope( + receiver, + new TestMessage(logicalId, 7, "atomic", sink.GetGrainId())); + + var result = await DeliverAsync(receiver, envelope); + var receiverState = await Fixture.WaitForEffectCountAsync(receiver, 1); + var sinkState = await Fixture.WaitForEffectCountAsync(sink, 1); + receiverState = await Fixture.WaitForOutboxCountAsync(receiver, 0); + + Assert.Equal(DeliveryStatus.Accepted, result.Status); + var effect = Assert.Single(receiverState.Effects); + Assert.Equal(new MessageEffect(logicalId, 1, 7, "atomic"), effect); + Assert.Equal(0, receiverState.InboxCount); + Assert.Equal(0, receiverState.OutboxCount); + Assert.Equal(effect, Assert.Single(sinkState.Effects)); + + var duplicate = await DeliverAsync(receiver, envelope); + Assert.Equal(DeliveryStatus.Duplicate, duplicate.Status); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + Assert.Equal(1, Assert.Single((await sink.GetSnapshotAsync()).Effects).Count); + } + + [Fact] + public async Task HandlerPreparationFailure_DeadLettersWithoutEffectsOrSinkDelivery() + { + var receiver = NewGrain(); + var sink = NewGrain(); + var envelope = CreateEnvelope( + receiver, + new TestMessage(HierarchicalKey.Create("preparation", "failure"), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); + + var accepted = await DeliverAsync(receiver, envelope); + var state = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); + + Assert.Equal(DeliveryStatus.Accepted, accepted.Status); + Assert.Empty(state.Effects); + Assert.Equal(0, state.InboxCount); + Assert.Equal(0, state.OutboxCount); + var deadLetter = Assert.Single(state.InboxDeadLetters); + Assert.Equal(envelope.Envelope.MessageId, deadLetter.MessageId); + Assert.Equal(1, deadLetter.AttemptCount); + Assert.Contains("Injected handler preparation failure", deadLetter.Reason, StringComparison.Ordinal); + Assert.Empty((await sink.GetSnapshotAsync()).Effects); + } + [Fact] + public async Task HandlerApplicationPreparation_AllowsIndependentWritesAndPreservesCancelledWriteWait() + { + var receiver = NewGrain(); + var sink = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(receiver); + var grain = Assert.IsType(context.GrainInstance); + var manager = context.ActivationServices.GetRequiredService(); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/admitted-outgoing"); + var envelope = CreateEnvelope(receiver, + new TestMessage(HierarchicalKey.Create("preparation", "admitted"), 91, "admitted", sink.GetGrainId()), "messages/admitted-outgoing"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var localPreparation = grain.GetSnapshotForTest(); + Assert.Empty(localPreparation.Effects); + Assert.Equal(1, localPreparation.InboxCount); + Assert.Equal(0, localPreparation.ProcessedMessageCount); + Assert.Equal(0, localPreparation.OutboxCount); + using var applicationPreparation = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/admitted-outgoing/application-preparation"); + handler.Release(); + await applicationPreparation.WaitUntilEnteredAsync(); + var preparing = grain.GetSnapshotForTest(); + Assert.Empty(preparing.Effects); + Assert.Equal(1, preparing.InboxCount); + Assert.Equal(0, preparing.ProcessedMessageCount); + Assert.Equal(0, preparing.OutboxCount); + await manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.DoesNotContain(grain.Captures, snapshot => snapshot.Effects.Count > 0 || snapshot.OutboxCount > 0); + + var writes = Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId())); + using var cancellation = new CancellationTokenSource(); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + Task completedWait; + try + { + applicationPreparation.Release(); + await storage.WaitUntilEnteredAsync(); + var captured = grain.GetSnapshotForTest(); + Assert.Equal(0, captured.InboxCount); + Assert.Equal(1, captured.ProcessedMessageCount); + Assert.Equal(1, captured.OutboxCount); + Assert.Single(captured.Effects); + Assert.All(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, receiver.GetGrainId()), + job => Assert.Equal(receiver.GetGrainId(), job.TargetGrainId)); + var canceledWait = manager.WriteStateAsync(cancellation.Token).AsTask(); + completedWait = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(canceledWait.IsCompleted); + Assert.False(completedWait.IsCompleted); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledWait); + Assert.False(completedWait.IsCompleted); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(JournalId.FromGrainId(receiver.GetGrainId()))); + Assert.Empty((await sink.GetSnapshotAsync()).Effects); + } + finally + { + storage.Release(); + } + await completedWait; + var delivered = await Fixture.WaitForEffectCountAsync(sink, 1); + var completed = await Fixture.WaitForOutboxCountAsync(receiver, 0); + Assert.Equal(Assert.Single(completed.Effects), Assert.Single(delivered.Effects)); + Assert.DoesNotContain(grain.Captures, snapshot => snapshot.Effects.Count > 0 && snapshot.ProcessedMessageCount == 0); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandlerCompletionPersistenceFailure_FreshActivationConvergesEffectDedupeAndSink(bool storageCommitted) + { + var receiver = NewGrain(); + var sink = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var oldContext = Fixture.GetGrainContext(receiver); + var oldGrain = Assert.IsType(oldContext.GrainInstance); + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/completion-outcome"); + var message = new TestMessage(HierarchicalKey.Create("completion", "outcome"), 92, "completion-outcome", sink.GetGrainId()); + var envelope = CreateEnvelope(receiver, message, "messages/completion-outcome"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (storageCommitted) + { + Fixture.Storage.FailAfterWrite(journal); + } + else + { + Fixture.Storage.FailWrite(journal); + } + handler.Release(); + Assert.IsType(await oldGrain.DeactivationFailure.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)); + await oldContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.Effects.Count == 1); + var delivered = await Fixture.WaitForEffectCountAsync(sink, 1); + completed = await Fixture.SnapshotProbe.WaitAsync(receiver.GetGrainId(), + snapshot => snapshot.ActivationId != before.ActivationId && snapshot.Effects.Count == 1 && snapshot.OutboxCount == 0); + + Assert.NotEqual(before.ActivationId, completed.ActivationId); + Assert.Equal(new MessageEffect(message.LogicalId, 1, message.Sequence, message.Value), Assert.Single(completed.Effects)); + Assert.Equal(Assert.Single(completed.Effects), Assert.Single(delivered.Effects)); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(0, completed.InboxCount); + Assert.Empty(completed.InboxDeadLetters); + Assert.Empty(completed.OutboxDeadLetters); + Assert.Equal(DeliveryStatus.Duplicate, (await DeliverAsync(receiver, envelope)).Status); + Assert.Equal(1, Assert.Single((await sink.GetSnapshotAsync()).Effects).Count); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs new file mode 100644 index 00000000000..7fe9e198021 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs @@ -0,0 +1,67 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Orleans.Serialization.Session; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicMessagingRoutingAndDeadLetterTests : MessagingBehaviorTestBase +{ + public PublicMessagingRoutingAndDeadLetterTests() : base(receiverOnly: false) + { + } + + [Fact] + public async Task HandlerNotFound_OutboxRetriesThenDeadLettersWithoutReceiverPersistence() + { + var sender = NewGrain(); + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await ReactivateAsync(receiver); + + await sender.SendAsync( + receiver.GetGrainId(), + "unknown/outbox-route", + NewMessage(72, "undeliverable")); + var senderState = await Fixture.WaitForDeadLetterCountAsync(sender, 1); + + Assert.Equal(0, senderState.OutboxCount); + var deadLetter = Assert.Single(senderState.OutboxDeadLetters); + Assert.Equal("unknown/outbox-route", deadLetter.Route); + Assert.Equal(3, deadLetter.AttemptCount); + Assert.Contains("No inbox handler", deadLetter.Reason, StringComparison.Ordinal); + var receiverState = await receiver.GetSnapshotAsync(); + Assert.Equal(0, receiverState.InboxCount); + Assert.Empty(receiverState.Effects); + Assert.Empty(receiverState.InboxDeadLetters); + } + + [Fact] + public async Task OutboxDeadLetterRemoval_IsDurable() + { + var sender = NewGrain(); + var receiver = NewGrain(); + await receiver.ConfigureHandlerAsync(false); + await ReactivateAsync(receiver); + var messageId = await sender.SendAsync( + receiver.GetGrainId(), + "unknown/removable-outbox-route", + NewMessage(73, "removable-undeliverable")); + _ = await Fixture.WaitForDeadLetterCountAsync(sender, 1); + + Assert.True(await sender.RemoveOutboxDeadLetterAsync(messageId)); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + Assert.False(await sender.RemoveOutboxDeadLetterAsync(messageId)); + + await sender.RequestDeactivationAsync(); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + +} diff --git a/test/Orleans.Messaging.Tests/Functional/PublicServiceCollectionMessagingCompositionTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicServiceCollectionMessagingCompositionTests.cs new file mode 100644 index 00000000000..01664a96263 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/PublicServiceCollectionMessagingCompositionTests.cs @@ -0,0 +1,11 @@ +using Orleans.Messaging.Tests.Support; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class PublicServiceCollectionMessagingCompositionTests() + : MessagingGrainTypeConfiguratorTests(new BootstrapClusterFixture(useServiceCollection: true)); diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs index b1dcce67742..33f65fe66b2 100644 --- a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -44,7 +44,7 @@ public void AddMessaging_RepeatedPublicRegistrationRetainsOneConfiguratorAndScop Assert.Equal(ServiceLifetime.Singleton, configurator.Lifetime); foreach (var contract in new[] { - typeof(MessagingPump), typeof(Inbox), typeof(IInbox), typeof(IMessagingDiagnostics) + typeof(MessagingPump), typeof(Inbox), typeof(Outbox), typeof(IInbox), typeof(IOutbox), typeof(IMessagingDiagnostics) }) { var descriptor = Assert.Single(services, descriptor => descriptor.ServiceType == contract && !descriptor.IsKeyedService); diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs new file mode 100644 index 00000000000..1076f11358b --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs @@ -0,0 +1,133 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class BootstrapDeliveryProbe : IOutgoingGrainCallFilter, ILoggerProvider +{ + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ConcurrentDictionary> _outputs = new(); + + private readonly ConcurrentDictionary _drains = new(); + + public Task WaitForDrainAsync(GrainId sender) => + _drains.GetOrAdd(sender, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)).Task.WaitAsync(TimeSpan.FromSeconds(30)); + + public ILogger CreateLogger(string categoryName) => new CompletionLogger(this, categoryName); + public void Dispose() { } + + private sealed class CompletionLogger(BootstrapDeliveryProbe owner, string category) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => category == "Orleans.Messaging.Outbox"; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (IsEnabled(logLevel) && eventId.Name == "LogDeliveryComplete" + && state is IEnumerable> fields + && fields.Any(field => field.Key == "RemainingCount" && field.Value is 0) + && ReceiverTestServices.CurrentGrainContext is { } context) + { + owner._drains.GetOrAdd(context.GrainId, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)).TrySetResult(); + } + } + } + + public async Task Invoke(IOutgoingGrainCallContext context) + { + if (context.TargetId == BootstrapState.OutputTarget && context.MethodName == nameof(IInboxExtension.DeliverAsync)) + { + await _release.Task.WaitAsync(context.Request.GetCancellationToken()); + } + await context.Invoke(); + } + + public void Release() => _release.TrySetResult(); + public Task<(HierarchicalKey MessageId, int Value)> WaitForOutputAsync(GrainId sender) => + GetOutput(sender).Task.WaitAsync(TimeSpan.FromSeconds(30)); + public void OnOutput(GrainId sender, HierarchicalKey messageId, int value) => GetOutput(sender).TrySetResult((messageId, value)); + private TaskCompletionSource<(HierarchicalKey MessageId, int Value)> GetOutput(GrainId sender) => + _outputs.GetOrAdd(sender, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)); +} + +public interface IBootstrapOutputGrain : IGrainWithStringKey +{ + Task GetMessageCountAsync(); +} + +[GrainType("bootstrap-output")] +public sealed class BootstrapOutputGrain : Grain, IBootstrapOutputGrain, IMessagingGrain, IInboxHandler, + IStateMachine, IDurableDictionaryCommandHandler +{ + private readonly Dictionary _values = []; + private readonly List<(GrainId Sender, HierarchicalKey MessageId, int Value)> _pending = []; + private (GrainId Sender, HierarchicalKey MessageId, int Value)[] _captured = []; + private readonly BootstrapDeliveryProbe _probe; + private readonly Orleans.Serialization.Session.SerializerSessionPool _sessions; + private readonly Orleans.Serialization.Serializer _senderSerializer; + private readonly IDurableDictionaryCommandCodec _codec; + + public BootstrapOutputGrain(IJournaledStateManager manager, IInbox inbox, BootstrapDeliveryProbe probe, + IDurableDictionaryCommandCodec codec, Orleans.Serialization.Session.SerializerSessionPool sessions, Orleans.Serialization.Serializer senderSerializer) + { + _sessions = sessions; + _senderSerializer = senderSerializer; + _probe = probe; + _codec = codec; + manager.RegisterStateMachine("bootstrap-output-values", this); + inbox.Register(this); + } + + public bool CanHandle(IInboxHandlerContext context) => true; + + + public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var value = (int)(TestApplicationProtocol.Read(_sessions, context.Message).Body + ?? throw new InvalidOperationException("The bootstrap output must contain an integer.")); + if (context.Message.Envelope.Headers is not { } headers || !headers.TryGetValue("sender", out var senderBytes)) + { + throw new InvalidOperationException("The bootstrap output must supply its sender."); + } + var sender = _senderSerializer.Deserialize(senderBytes); + _values.Add(context.Message.Envelope.MessageId, value); + _pending.Add((sender, context.Message.Envelope.MessageId, value)); + context.Complete(); + return ValueTask.CompletedTask; + } + + public Task GetMessageCountAsync() => Task.FromResult(_values.Count); + public void WritePendingEntries(JournalStreamWriter writer) + { + _captured = _pending.ToArray(); + _pending.Clear(); + foreach (var item in _captured) _codec.WriteSet(item.MessageId, item.Value, writer); + } + public void WriteSnapshot(JournalStreamWriter writer) + { + _captured = _pending.ToArray(); + _pending.Clear(); + _codec.WriteSnapshot(_values, writer); + } + public void OnWriteCompleted() + { + foreach (var output in _captured) _probe.OnOutput(output.Sender, output.MessageId, output.Value); + _captured = []; + } + public void Reset(JournalStreamWriter writer) + { + _values.Clear(); + _pending.Clear(); + _captured = []; + } + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); + + public void ApplySet(HierarchicalKey key, int value) => _values[key] = value; + public void ApplyRemove(HierarchicalKey key) => _values.Remove(key); + public void ApplyClear() => _values.Clear(); + public void Reset(int capacityHint) => _values.Clear(); +} diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs new file mode 100644 index 00000000000..f9c744c42d4 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -0,0 +1,321 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Orleans.Concurrency; +using Orleans.Journaling; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Serialization.Invocation; + +namespace Orleans.Messaging.Tests.Support; + +public sealed class BootstrapProbe +{ + private readonly ConcurrentDictionary> _activations = new(); + public void Add(BootstrapObservation observation) => + _activations.GetOrAdd(observation.Context.GrainId, static _ => new()).Enqueue(observation); + public BootstrapObservation[] Get(GrainId id) => _activations[id].ToArray(); +} + +public sealed class BootstrapObservation : IDisposable +{ + public BootstrapObservation(IGrainContext context, BootstrapProbe probe) + { + Context = context; + probe.Add(this); + } + + public IGrainContext Context { get; } + public object? ConstructedGrain { get; private set; } + public bool InstanceAvailableInConstructor { get; private set; } + public IJournaledStateManager? Manager { get; set; } + public IDurableValue? Value { get; set; } + internal Inbox? Inbox { get; set; } + public IOutbox? Outbox { get; set; } + public Exception? ExpectedFailure { get; set; } + public int Activations { get; set; } + public int GrainDisposals { get; set; } + public int Disposals { get; private set; } + + public void Constructed(object grain) + { + ConstructedGrain = grain; + InstanceAvailableInConstructor = Context.GrainInstance is not null; + } + + public void Dispose() => Disposals++; +} + +public sealed class BootstrapState : IInboxHandler, IDisposable +{ + public const string Route = "bootstrap"; + public static GrainId OutputTarget { get; } = GrainId.Create("bootstrap-output", "capture"); + private readonly HandlerProbe _handlers; + public BootstrapState(BootstrapObservation observation, IJournaledStateManager manager, HandlerProbe handlers, + [FromKeyedServices("bootstrap-value")] IDurableValue value, IGrainContext context, IOutbox outbox) + { + _handlers = handlers; + Observation = observation; + observation.Manager = manager; + observation.Value = value; + observation.Inbox = context.ActivationServices.GetRequiredService(); + observation.Outbox = outbox; + observation.Inbox.Register(this); + } + + public BootstrapObservation Observation { get; } + public int MessagingStateCountAtActivation { get; private set; } + public object? GrainAtActivation { get; private set; } + public int ActivationValue { get; private set; } + public int HandlerCalls { get; private set; } + internal IInboxHandler? HandlerOverride { get; set; } + public int Disposals { get; private set; } + public Task Read() => Task.FromResult(Observation.Value!.Value); + public async Task Set(int value) + { + Observation.Value!.Value = value; + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendValue(int value) + { + var context = Observation.Context; + var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, + HierarchicalKey.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public async Task SendSynchronousValue(int value) + { + var context = Observation.Context; + var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, + HierarchicalKey.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + context.GrainId, OutputTarget, "output", value); + Observation.Value!.Value = value; + Observation.Outbox!.Send(envelope); + await Observation.Manager!.WriteStateAsync(CancellationToken.None); + } + public Task Persist() => Observation.Manager!.WriteStateAsync(CancellationToken.None).AsTask(); + public Task Activate() + { + Observation.Activations++; + GrainAtActivation = Observation.Context.GrainInstance; + MessagingStateCountAtActivation = ReadMessagingStates(Observation.Manager!).Count(); + ActivationValue = Observation.Value!.Value; + return Task.CompletedTask; + } + + public bool CanHandle(IInboxHandlerContext context) => true; + + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (HandlerOverride is { } handler) + { + await handler.HandleAsync(context, cancellationToken); + return; + } + if (_handlers.TryGet(Observation.Context.GrainId, Route, out var barrier)) + { + barrier.Entered.TrySetResult(); + await barrier.Continue.Task.WaitAsync(cancellationToken); + } + var value = Observation.Value!.Value + 1; + var outgoing = OutboxTestMessages.CreateApplication(Observation.Context.ActivationServices, + context.Message.Envelope.MessageId.CreateChildKey("output"), Observation.Context.GrainId, OutputTarget, "output", value); + Observation.Value.Value = value; + Observation.Outbox!.Send(outgoing); + HandlerCalls++; + context.Complete(); + } + public void Dispose() => Disposals++; + public static IEnumerable ReadMessagingStates(IJournaledStateManager manager) + { + foreach (var name in new[] { MessagingStateNames.Inbox, MessagingStateNames.Outbox }) + { + if (manager.TryGetStateMachine(name, out var state)) yield return state; + } + } + +} + +public interface IBootstrapTestGrain : IGrainWithGuidKey +{ + Task GetValueAsync(); + Task SetValueAsync(int value); + Task SendValueAsync(int value); + Task SendSynchronousValueAsync(int value); + Task PersistAsync(); + Task DeactivateAsync(); +} +public interface IMarkedBootstrapTestGrain : IBootstrapTestGrain, IMessagingGrain; +public interface IGenericBootstrapTestGrain : IBootstrapTestGrain; + +public abstract class BootstrapGrainBase : Grain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + protected BootstrapGrainBase(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} +public sealed class PlainBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public abstract class ApplicationBootstrapBase(BootstrapState state) : BootstrapGrainBase(state), IMessagingGrain; +public sealed class ApplicationBootstrapGrain(BootstrapState state) : ApplicationBootstrapBase(state); +public sealed class InterfaceBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IMarkedBootstrapTestGrain; +public sealed class GenericBootstrapGrain(BootstrapState state) : BootstrapGrainBase(state), IGenericBootstrapTestGrain, IMessagingGrain; + +public class DurableBootstrapGrain : DurableGrain, IBootstrapTestGrain, IDisposable +{ + private readonly BootstrapState _state; + public DurableBootstrapGrain(BootstrapState state) + { + _state = state; + state.Observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) => _state.Activate(); + public Task GetValueAsync() => _state.Read(); + public Task SetValueAsync(int value) => _state.Set(value); + public Task SendValueAsync(int value) => _state.SendValue(value); + public Task SendSynchronousValueAsync(int value) => _state.SendSynchronousValue(value); + public Task PersistAsync() => _state.Persist(); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } + public void Dispose() => _state.Observation.GrainDisposals++; +} + +public sealed class MarkedDurableBootstrapGrain(BootstrapState state) : DurableBootstrapGrain(state), IMessagingGrain; + +public interface IBootstrapControlGrain : IGrainWithGuidKey { Task PingAsync(); } +public interface IInterleavingBootstrapControlGrain : IBootstrapControlGrain +{ + [AlwaysInterleave] Task InterleaveAsync(); +} +public abstract class BootstrapControlGrain : Grain, IBootstrapControlGrain, IDisposable +{ + protected BootstrapObservation Observation { get; } + protected BootstrapControlGrain(BootstrapObservation observation) + { + Observation = observation; + observation.Constructed(this); + } + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + Observation.Activations++; + return Task.CompletedTask; + } + public Task PingAsync() => Task.CompletedTask; + public void Dispose() => Observation.GrainDisposals++; +} +public sealed class EagerBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation); +public sealed class FailingBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +public sealed class UnselectedEndpointBootstrapGrain : BootstrapControlGrain +{ + public UnselectedEndpointBootstrapGrain(BootstrapObservation observation, IGrainContext context) : base(observation) => + observation.Inbox = context.ActivationServices.GetRequiredService(); +} +public sealed class JournalOnlyBootstrapGrain : BootstrapControlGrain +{ + public JournalOnlyBootstrapGrain(BootstrapObservation observation, IJournaledStateManager manager, + [FromKeyedServices("bootstrap-journal-only")] IDurableValue value) : base(observation) + { + observation.Manager = manager; + observation.Value = value; + } +} +[Reentrant] +public sealed class ReentrantBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[StatelessWorker] +public sealed class StatelessBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain; +[MayInterleave(nameof(Interleave))] +public sealed class MayInterleaveBootstrapGrain(BootstrapObservation observation) : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} +public sealed class AlwaysInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IInterleavingBootstrapControlGrain, IMessagingGrain +{ + public Task InterleaveAsync() => Task.CompletedTask; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.Reentrant, "true")] +public sealed class MetadataReentrantBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.MayInterleavePredicate, nameof(Interleave))] +public sealed class MetadataMayInterleaveBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain +{ + public static bool Interleave(IInvokable request) => true; +} + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, "StatelessWorkerPlacement")] +public sealed class MetadataStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.StatelessPlacementAlias)] +public sealed class AliasedStatelessPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, nameof(RandomPlacement))] +public sealed class MetadataOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +[ExecutionProperty(WellKnownGrainTypeProperties.PlacementStrategy, BootstrapClusterFixture.OrdinaryPlacementAlias)] +public sealed class AliasedOrdinaryPlacementBootstrapGrain(BootstrapObservation observation) + : BootstrapControlGrain(observation), IMessagingGrain; + +public sealed class BootstrapClusterFixture : MessagingClusterFixture +{ + public const string StatelessPlacementAlias = "bootstrap-worker-alias"; + public const string OrdinaryPlacementAlias = "bootstrap-directory-alias"; + + public BootstrapClusterFixture(bool useServiceCollection = false) + : base(1, receiverOnly: false, useServiceCollection: useServiceCollection) + { + } + + public BootstrapProbe Probe { get; } = new(); + public BootstrapDeliveryProbe Delivery { get; } = new(); + protected override void ConfigureServices(IServiceCollection services) + { + services.AddScoped(provider => provider.GetRequiredKeyedService>("orleans-binary")); + services.AddKeyedSingleton(StatelessPlacementAlias, new StatelessWorkerAttribute(1).PlacementStrategy); + services.AddKeyedSingleton(OrdinaryPlacementAlias, new RandomPlacement()); + services.AddSingleton(Probe); + services.AddSingleton(Delivery); + services.AddSingleton(Delivery); + services.AddSingleton(Delivery); + services.AddScoped(); + services.AddScoped(); + services.RemoveAll(); + services.AddScoped(provider => + { + var inbox = ActivatorUtilities.CreateInstance(provider); + if (provider.GetRequiredService().GrainInstance is FailingBootstrapGrain) + { + var observation = provider.GetRequiredService(); + observation.Manager = provider.GetRequiredService(); + observation.Inbox = inbox; + observation.Outbox = provider.GetRequiredService(); + observation.ExpectedFailure = new IOException("Expected bootstrap runtime construction failure."); + throw observation.ExpectedFailure; + } + + return inbox; + }); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs index 0bf0554444b..f2870bcca47 100644 --- a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs +++ b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs @@ -9,12 +9,14 @@ internal static class InboxTestAccess public static int Count(this IOutbox outbox) => outbox switch { + Outbox value => value.Count, JournaledTestOutbox value => value.Count, _ => throw new NotSupportedException(outbox.GetType().FullName) }; public static IEnumerable Messages(this IOutbox outbox) => outbox switch { + Outbox value => value.MessageState.Values.Select(static entry => entry.Message), JournaledTestOutbox value => value.Messages, _ => throw new NotSupportedException(outbox.GetType().FullName) }; diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 75fbe07338f..11d24f029a3 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -23,6 +23,7 @@ public interface IMessagingTestGrain : IGrainWithGuidKey Task AcceptAndDeactivateAsync(InboxMessage envelope); Task ConfigureHandlerAsync(bool enabled); Task RemoveInboxDeadLetterAsync(MessageId messageId); + Task RemoveOutboxDeadLetterAsync(MessageId messageId); Task GetSnapshotAsync(); Task RequestDeactivationAsync(); Task SetControlEnvelopeAsync(InboxMessage envelope); @@ -225,6 +226,17 @@ public async Task RemoveInboxDeadLetterAsync(MessageId messageId) return true; } + public async Task RemoveOutboxDeadLetterAsync(HierarchicalKey messageId) + { + if (!_diagnostics.RemoveOutboxDeadLetter(messageId)) + { + return false; + } + + await WriteStateAsync(); + return true; + } + public Task GetSnapshotAsync() => Task.FromResult(CreateSnapshot()); internal EndpointSnapshot GetSnapshotForTest() => CreateSnapshot(); @@ -428,7 +440,7 @@ private EndpointSnapshot CreateSnapshot() => _maxConcurrentHandlers, _effects.Values.OrderBy(static effect => effect.Sequence).ToArray(), _diagnostics.InboxDeadLetters.Select(ToSnapshot).ToArray(), - [], + _diagnostics.OutboxDeadLetters.Select(ToSnapshot).ToArray(), InboxTestAccess.GetProcessedCount(_inboxState)); private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => @@ -439,6 +451,14 @@ private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => deadLetter.AttemptCount, deadLetter.DeadLetteredAt); + private DeadLetterSnapshot ToSnapshot(DeadLetter deadLetter) => + new( + deadLetter.Message.Envelope.MessageId, + TestApplicationProtocol.Read(_sessions, deadLetter.Message).Route, + deadLetter.Reason, + deadLetter.AttemptCount, + deadLetter.DeadLetteredAt); + private sealed class ApplicationDispatcher(MessagingTestGrain owner) : IInboxHandler { public bool CanHandle(InboxMessage message) => true; diff --git a/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs b/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs new file mode 100644 index 00000000000..20aa12e5e24 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs @@ -0,0 +1,11 @@ +namespace Orleans.Messaging.Tests.Support; + +internal sealed class OutboxStateTestAccess(IOutbox outbox) +{ + private readonly Outbox _outbox = (Outbox)outbox; + + public IEnumerable AttemptKeys => _outbox.MessageState.Keys; + public IEnumerable DeadLetterKeys => _outbox.DeadLetterState.Keys; + public OutboxMessageState Attempt(HierarchicalKey id) => _outbox.MessageState[id].State; + public OutboxDeadLetter DeadLetter(HierarchicalKey id) => _outbox.DeadLetterState[id]; +} diff --git a/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs new file mode 100644 index 00000000000..a2188e7b0ed --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs @@ -0,0 +1,31 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Serialization; +using Orleans.Serialization.Session; + +namespace Orleans.Messaging.Tests.Support; + +internal static class OutboxTestMessages +{ + public static OutboxMessage Create(HierarchicalKey messageId, GrainId receiver, ReadOnlyMemory payload, + GrainId? sender, string? subject, Serializer senderSerializer) => new() + { + ReceiverId = receiver, + Envelope = new Envelope + { + MessageId = messageId, + Payload = payload.ToArray(), + Subject = subject, + Headers = sender is { } senderId + ? new Dictionary(StringComparer.Ordinal) { ["sender"] = senderSerializer.SerializeToArray(senderId) } + : null + } + }; + + public static OutboxMessage CreateApplication(IServiceProvider services, HierarchicalKey messageId, + GrainId sender, GrainId receiver, string subject, object? body) => Create(messageId, receiver, + TestApplicationProtocol.Encode(services.GetRequiredService(), new TestApplicationMessage(subject, body)), + sender, subject, services.GetRequiredService>()); + + public static string? Subject(Envelope envelope) => envelope.Subject; +} diff --git a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs new file mode 100644 index 00000000000..fc6b0eb4174 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs @@ -0,0 +1,115 @@ +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Journaling; +using Orleans.Runtime; + +namespace Orleans.Messaging.Tests.Support; + +public interface IRawPayloadTestGrain : IGrainWithGuidKey +{ + Task ConfigureForwardAsync(GrainId target, int copies = 1); + Task AcceptAndDeactivateAsync(InboxMessage message); + Task GetSnapshotAsync(); + Task RequestDeactivationAsync(); +} + +[GenerateSerializer] +public sealed record RawPayloadEffect([property: Id(0)] HierarchicalKey MessageId, [property: Id(1)] string Bytes, [property: Id(2)] int Count); + +[GenerateSerializer] +public sealed record RawPayloadSnapshot([property: Id(0)] Guid ActivationId, [property: Id(1)] int InboxCount, + [property: Id(2)] int OutboxCount, [property: Id(3)] int ProcessedCount, [property: Id(4)] RawPayloadEffect[] Effects); + +// The application protocol is arbitrary bytes. Orleans codecs are only used for journal commands, +// business state and RPC results, never to encode/decode the messaging payload. +public sealed class RawPayloadTestGrain : DurableGrain, IRawPayloadTestGrain, IInboxHandler +{ + public const string HandlerBarrier = "raw-payload"; + private readonly Inbox _inbox; + private readonly IOutbox _outbox; + private readonly IDurableDictionary _effects; + private readonly IDurableValue _forward; + private readonly IDurableValue _forwardCopies; + internal int ExpectedAcknowledgedEffects { get; set; } = 1; + private readonly HandlerProbe _handlers; + private readonly Guid _activationId = Guid.NewGuid(); + private readonly TaskCompletionSource _acknowledged = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal Task Acknowledged => _acknowledged.Task; + + public RawPayloadTestGrain(IGrainContext context, IOutbox outbox, IJournaledStateManager manager, + [FromKeyedServices("raw-effects")] IDurableDictionary effects, + [FromKeyedServices("raw-forward")] IDurableValue forward, + [FromKeyedServices("raw-forward-copies")] IDurableValue forwardCopies, + HandlerProbe handlers) + { + _inbox = context.ActivationServices.GetRequiredService(); + _outbox = outbox; + _effects = effects; + _forward = forward; + _forwardCopies = forwardCopies; + _handlers = handlers; + _inbox.Register(this); + new JournalProbeState(manager).WriteCompleted = () => + { + if (_inbox.Count == 0 && _outbox.Count() == 0 && _effects.Count == ExpectedAcknowledgedEffects) + { + _acknowledged.TrySetResult(GetSnapshotForTest()); + } + }; + } + + public async Task ConfigureForwardAsync(GrainId target, int copies = 1) + { + _forward.Value = target; + _forwardCopies.Value = copies; + await WriteStateAsync(); + } + + public async Task AcceptAndDeactivateAsync(InboxMessage message) + { + var result = await ((IInboxExtension)_inbox).DeliverAsync(message); + AcceptedSnapshot = GetSnapshotForTest(); + DeactivateOnIdle(); + return result; + } + + public bool CanHandle(IInboxHandlerContext context) => true; + + public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) + { + if (_handlers.TryGet(this.GetGrainId(), HandlerBarrier, out var barrier)) + { + barrier.Entered.TrySetResult(); + await barrier.Continue.Task.WaitAsync(cancellationToken); + } + cancellationToken.ThrowIfCancellationRequested(); + var input = context.Message.Envelope; + var bytes = Convert.ToBase64String(input.Payload); + var outgoing = _forward.Value.IsDefault + ? [] + : Enumerable.Range(0, _forwardCopies.Value).Select(index => new OutboxMessage + { + Envelope = index == 0 + ? input + : input with { MessageId = input.MessageId.CreateChildKey(index.ToString(CultureInfo.InvariantCulture)) }, + ReceiverId = _forward.Value + }).ToArray(); + _effects.TryGetValue(input.MessageId, out var prior); + _effects[input.MessageId] = new RawPayloadEffect(input.MessageId, bytes, (prior?.Count ?? 0) + 1); + foreach (var envelope in outgoing) + { + _outbox.Send(envelope); + } + context.Complete(); + } + + internal RawPayloadSnapshot? AcceptedSnapshot { get; private set; } + + public Task GetSnapshotAsync() => Task.FromResult(GetSnapshotForTest()); + internal RawPayloadSnapshot GetSnapshotForTest() => new(_activationId, _inbox.Count, _outbox.Count(), InboxTestAccess.GetProcessedCount(_inbox), _effects.Values.ToArray()); + public Task RequestDeactivationAsync() + { + DeactivateOnIdle(); + return Task.CompletedTask; + } +} From cd6a24052665fa05f32764faa0a94379400684f9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 18:21:09 -0700 Subject: [PATCH 64/95] fix(messaging): schedule outbox retries on the durable jobs clock --- src/Orleans.Messaging/Outbox.cs | 3 ++- .../Functional/MultiSiloMessagingFailoverTests.cs | 2 +- .../Functional/PublicInboxHandlerTransactionTests.cs | 10 +++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 23d34c210cc..aed3efd6e9e 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -34,7 +34,8 @@ public Outbox(MessagingPump pump) private MessagingShared Shared => _pump.Shared; private DurableMessagingOptions Options => _pump.Shared.Options; private ILogger Logger => _pump.Shared.OutboxLogger; - private TimeProvider TimeProvider => _pump.Shared.TimeProvider; + // Delivery retries, expiry, and retention follow the durable jobs clock which schedules outbox wake-ups. + private TimeProvider TimeProvider => _pump.Shared.JobTimeProvider; private GrainId GrainId => _pump.GrainContext.GrainId; internal IReadOnlyDictionary MessageState => _messages; diff --git a/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs index 20b5d057fdc..b4ff786accb 100644 --- a/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs @@ -22,7 +22,7 @@ public async Task ReceiverSiloStops_DuringBlockedHandler_NewActivationRecoversFr using var barrier = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failover"); var sender = fixture.Client.GetGrain(Guid.NewGuid()); var logicalId = HierarchicalKey.Create("failover", Guid.NewGuid().ToString("N")); - fixture.JobManagerProbe.DuplicateNext(MessagingPump.JobName); + fixture.JobManagerProbe.DuplicateNext(MessagingPump.JobName, receiver.GetGrainId()); await sender.SendAsync( receiver.GetGrainId(), diff --git a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs index 48a5c193076..c03629d154a 100644 --- a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs @@ -38,9 +38,13 @@ public async Task ArbitraryRawPayload_AcceptanceReplayForwardAndBusinessCompleti Payload = bytes, Headers = new Dictionary(StringComparer.Ordinal) { ["attachment"] = [0xff, 0x80, 0x01] } }; - Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync( - new InboxMessage { Envelope = envelope })).Status); - await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + // Deactivation cancels any handler the original activation starts, leaving the message to recovery. + using (Fixture.HandlerProbe.Arm(receiver.GetGrainId(), RawPayloadTestGrain.HandlerBarrier)) + { + Assert.Equal(DeliveryStatus.Accepted, (await receiver.AcceptAndDeactivateAsync( + new InboxMessage { Envelope = envelope })).Status); + await original.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } var accepted = Assert.IsType(Assert.IsType(original.GrainInstance).AcceptedSnapshot); Assert.Equal(1, accepted.InboxCount); Assert.Empty(accepted.Effects); From 81d23c627d54f363127a37ff6c2c19b163a2b782 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 20:47:59 -0700 Subject: [PATCH 65/95] fix(messaging): validate recovered outbox dead-letter keys --- src/Orleans.Messaging/Outbox.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index aed3efd6e9e..212cf325cb0 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -82,7 +82,14 @@ private void OnRecoveryCompleted() throw new InvalidOperationException("The recovered outbox key does not match its command identity."); } } - foreach (var letter in _deadLetters.Values) EnvelopeValidation.Validate(letter.Message); + foreach (var (key, letter) in _deadLetters) + { + EnvelopeValidation.Validate(letter.Message); + if (key != letter.Message.Envelope.MessageId) + { + throw new InvalidOperationException("The recovered outbox dead-letter key does not match its message identity."); + } + } _capturedSequence = _durableSequence = _lastSequence; } From 41628d7e9c4cec093f96a2076031366585fb25ea Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 21:57:45 -0700 Subject: [PATCH 66/95] feat(messaging): deliver scheduled outbox messages immediately The outbox hands messages with DeliverAt to the receiver as soon as they are durable; the receiving inbox holds them until they are due. The pump reads the Durable Jobs clock once per run and passes that time to both queues when computing the next wake-up. --- src/Orleans.Messaging/MessagingPump.cs | 2 +- src/Orleans.Messaging/Outbox.cs | 4 +-- .../MessagingCommandSerializationTests.cs | 9 ++++-- .../Functional/JsonJournalMessagingTests.cs | 28 +++++++++++++++++ .../Functional/ScheduledDeliveryTests.cs | 30 ++++++++++++++++++- 5 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 4d180561dcc..395c1b627d9 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -173,7 +173,7 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte var now = Shared.JobTimeProvider.GetUtcNow(); var inboxDelay = Inbox.GetNextWakeupDelay(now); - var outboxDelay = Outbox.GetNextWakeupDelay(); + var outboxDelay = Outbox.GetNextWakeupDelay(now); if (inboxDelay is null && outboxDelay is null) { CompleteWakeup(jobId); diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 212cf325cb0..b76913b2a20 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -370,10 +370,10 @@ private DateTimeOffset GetNextAttemptAt(OutboxMessageState state, DateTimeOffset /// /// Returns the delay until the next delivery attempt is due, or when the outbox is empty. /// - internal TimeSpan? GetNextWakeupDelay() + /// The current Durable Jobs time. + internal TimeSpan? GetNextWakeupDelay(DateTimeOffset now) { if (Count == 0) return null; - var now = TimeProvider.GetUtcNow(); var next = DateTimeOffset.MaxValue; foreach (var entry in _messages.Values) { diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 7510d9e00f8..a31cee8dd1d 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -71,11 +71,15 @@ public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWh Assert.DoesNotContain("DeliverAt", immediateJson, StringComparison.Ordinal); Assert.Null(JsonSerializer.Deserialize(immediateJson, InboxJsonContext.Default.InboxCommand)!.Message!.Value.Envelope.DeliverAt); Assert.Null(serializer.Deserialize(serializer.SerializeToArray(immediate))!.Message!.Value.Envelope.DeliverAt); + Assert.DoesNotContain("DeliverAt", + JsonSerializer.Serialize(new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("immediate"), State = new() { EnqueuedAt = Now } }, + OutboxJsonContext.Default.OutboxCommand), + StringComparison.Ordinal); } private static InboxMessage InboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt) }; - private static OutboxMessage OutboxMessage(string id) => new() { Envelope = Envelope(id), ReceiverId = Receiver }; + private static OutboxMessage OutboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt), ReceiverId = Receiver }; private static Envelope Envelope(string id, DateTimeOffset? deliverAt = null) => new() { @@ -121,6 +125,7 @@ private static OutboxCommand[] CreateOutboxCommands() return [ new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() { EnqueuedAt = Now } }, + new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() { EnqueuedAt = Now } }, new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", EnqueuedAt = Now } }, new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "dead"), DeadLetter = deadLetter }, new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [HierarchicalKey.Create("orders", "dead")] }, @@ -129,7 +134,7 @@ private static OutboxCommand[] CreateOutboxCommands() Kind = OutboxCommandKind.Snapshot, Snapshot = new() { - Messages = [new() { Message = OutboxMessage("pending"), State = new() { AttemptCount = 1, EnqueuedAt = Now } }], + Messages = [new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, EnqueuedAt = Now } }], DeadLetters = [deadLetter] } } diff --git a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs index 28d4fc3932e..4e936f98f3c 100644 --- a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs @@ -61,6 +61,34 @@ public async Task CustomInboxHandlers_FirstHandlerWhichCanHandleReceivesMessage( Assert.Equal(new[] { CustomHandlerGrain.FirstMatch }, await Probe.WaitAsync(receiver.GetGrainId(), 1, Token)); } + [Fact] + public async Task JsonJournal_RecoversScheduledInboxMessageWithItsDeliverAt() + { + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + var messageId = HierarchicalKey.Create("json", "scheduled", Guid.NewGuid().ToString("N")); + var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); + var deliverAt = new DateTimeOffset(DateTime.UtcNow.AddDays(1).Ticks, TimeSpan.Zero).ToOffset(TimeSpan.FromHours(5)); + var scheduled = type.Create(messageId, receiver.GetGrainId(), 7, deliverAt).ToInboxMessage(); + var extension = receiver.AsReference(); + + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); + Assert.True(Cluster.TryGetGrainContext(receiver.GetGrainId(), out var receiverContext)); + await receiver.DeactivateAsync(); + await receiverContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + // Redelivery replays the JSON journal; pending-command equivalence requires the recovered DeliverAt. + Assert.Equal(DeliveryStatus.Duplicate, (await extension.DeliverAsync(scheduled, Token)).Status); + var conflicting = type.Create(messageId, receiver.GetGrainId(), 7, deliverAt.AddSeconds(1)).ToInboxMessage(); + await Assert.ThrowsAsync(() => extension.DeliverAsync(conflicting, Token).AsTask()); + Assert.True(Cluster.TryGetGrainContext(receiver.GetGrainId(), out var recoveredContext)); + Assert.NotSame(receiverContext, recoveredContext); + var inbox = recoveredContext.ActivationServices.GetRequiredService(); + var recovered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + recoveredContext.Scheduler.QueueAction(() => recovered.SetResult(inbox.MessageState[messageId].Message.Envelope.DeliverAt)); + Assert.Equal(deliverAt, await recovered.Task.WaitAsync(TimeSpan.FromSeconds(30), Token)); + Assert.Empty(Probe.Get(receiver.GetGrainId())); + } + public sealed class Fixture : IAsyncLifetime { public JsonMessagingProbe Probe { get; } = new(); diff --git a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs index 60bcc531828..507db0e4920 100644 --- a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs @@ -200,6 +200,34 @@ public async Task ScheduledMessage_RedeliveryCoalescesAndDifferentDeliverAtConfl Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); } + [Fact] + public async Task SelfSentScheduledMessage_IsDeliveredImmediatelyAndProcessedWhenDue() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var deliverAt = JobClock.GetUtcNow() + Delay; + var message = NewMessage(40, "self"); + var outgoing = TestApplicationProtocol.Create(Sessions, grain.GetGrainId(), grain.GetGrainId(), "messages/self", message); + outgoing = new OutboxMessage { ReceiverId = outgoing.ReceiverId, Envelope = outgoing.Envelope with { DeliverAt = deliverAt } }; + + await grain.StageOutputAsync(outgoing); + await grain.RetryWriteStateAsync(); + + // The outbox hands the message to its own inbox without waiting for the due time. + var delivered = await Fixture.SnapshotProbe.WaitAsync( + grain.GetGrainId(), static snapshot => snapshot.InboxCount == 1 && snapshot.OutboxCount == 0); + Assert.Empty(delivered.Effects); + await AssertWakeupAtAsync(grain, deliverAt); + Assert.Empty((await grain.GetSnapshotAsync()).Effects); + + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal("self", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + } + [Fact] public async Task ScheduledMessage_LeavesActivationCollectableUntilDue() { @@ -232,7 +260,7 @@ private async Task AssertWakeupAtAsync(IMessagingTestGrain grain, DateTimeOffset private static InboxMessage Schedule(InboxMessage message, DateTimeOffset? deliverAt) => new() { Envelope = message.Envelope with { DeliverAt = deliverAt } }; - private sealed class ScheduledDeliveryClusterFixture() : MessagingClusterFixture(1, receiverOnly: true) + private sealed class ScheduledDeliveryClusterFixture() : MessagingClusterFixture(1, receiverOnly: false) { public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); From 64e083baae41751b2a915fa31c418188d05b45f6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 12 Oct 2026 23:09:32 -0700 Subject: [PATCH 67/95] docs(messaging): document terminal outbox delivery outcomes --- src/Orleans.Messaging/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 33ee10f5b8c..6ff719b607b 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -145,10 +145,13 @@ write completes. Each receiver returns a `DeliveryResult`: - `Backpressured`: the inbox holds `MaxCapacity` pending messages without `DeliverAt`. The inbox admits scheduled messages regardless of `MaxCapacity`. - `HandlerNotFound`: the receiver has no registered handlers. +- `DeadLettered`: the receiver moved the message to its dead-letter store. -The outbox removes accepted and duplicate messages, and retries other outcomes with -exponential backoff. It dead-letters a message after `MaxDeliveryAttempts` or once it -is older than `MaxOutboxRetryAge`. Delivery order is unspecified. +`Accepted`, `Duplicate`, and `DeadLettered` are terminal: the outbox removes the +message. The outbox retries `Backpressured`, `HandlerNotFound`, and transport +failures with exponential backoff. It dead-letters a message after +`MaxDeliveryAttempts` or once it is older than `MaxOutboxRetryAge`. Delivery order +is unspecified. Each activation runs one messaging pump on its scheduler, processing inbox batches and delivering outbox batches. One durable job per grain wakes the pump while either From 77cc003a82758dd2b855340ebf0fbe30be90ec46 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 00:05:41 -0700 Subject: [PATCH 68/95] fix(messaging): bind IOutbox to the delivered outbox AddMessaging replaces any earlier IOutbox registration with the scoped Outbox because the messaging pump delivers from the concrete Outbox. --- .../Hosting/MessagingExtensions.cs | 2 +- .../Hosting/PublicMessagingRegistrationTests.cs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs index 78ef142cbdd..f8ce5b3d1eb 100644 --- a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -79,7 +79,7 @@ public static IServiceCollection AddMessaging(this IServiceCollection services, services.TryAddScoped(); services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); services.TryAddScoped(); - services.TryAddScoped(sp => sp.GetRequiredService()); + services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); services.TryAddScoped(); services.TryAddEnumerable( ServiceDescriptor.Singleton()); diff --git a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs index 33f65fe66b2..02a5ce2d9b9 100644 --- a/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs +++ b/test/Orleans.Messaging.Tests/Hosting/PublicMessagingRegistrationTests.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; +using NSubstitute; using Orleans.Hosting; using Orleans.Journaling; using Orleans.Messaging.Configuration; @@ -133,6 +134,22 @@ public async Task AddMessaging_BindsInboxContractToDeliveredInboxOverEarlierRegi Assert.Same(scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService()); } + [Fact] + public async Task AddMessaging_BindsOutboxContractToDeliveredOutboxOverEarlierRegistration() + { + var services = new ServiceCollection(); + services.AddScoped(static _ => Substitute.For()); + services.AddMessaging(); + + // Outbox construction needs an activation, so substitute an uninitialized scoped instance to observe the binding. + services.Replace(ServiceDescriptor.Scoped(static _ => (Outbox)RuntimeHelpers.GetUninitializedObject(typeof(Outbox)))); + await using var provider = services.BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + Assert.Single(services, static descriptor => descriptor.ServiceType == typeof(IOutbox)); + Assert.Same(scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService()); + } + [Fact] public async Task AddMessaging_DoesNotReplaceUnrelatedConstructionErrors() { From 5a542518c7bee52886159e19f7d3fa11d27963ce Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 07:55:46 -0700 Subject: [PATCH 69/95] feat(messaging): stage self-addressed messages directly in the inbox IOutbox.Send previously staged an outbox entry for every receiver, including the sending grain. After that write completed, the pump delivered the message to its own inbox in-process, the inbox journaled the acceptance, and the outbox journaled the delivery: three writes, with the message stored twice in between. Send now stages acceptance in the grain's own inbox when the receiver is the current grain. Inbox.StageLocal applies the delivery identity rules synchronously and without writing: a completed ID within the deduplication window or an equivalent pending command is a duplicate, and a different pending command with the same ID throws InvalidOperationException. The Accept command joins the caller's next journal write, so the message becomes durable atomically with the caller's other staged state in one write, and existing sequence tracking defers processing until that write completes. The write's durable wake-up starts the pump once the activation is servicing calls, so activation-time sends never run handlers during activation. A synchronous send cannot report backpressure or a missing handler, so self-addressed messages are always admitted. Unscheduled ones count toward MaxCapacity for deliveries from other grains until they complete, and processing waits for handler registration. A self-send counts as sent and as received (accepted or duplicate) and never affects outbox depth or delivery metrics. The outbox therefore never holds self-addressed messages: the in-process self-delivery paths are removed and recovery rejects self-addressed outbox entries and dead letters. --- .../Configuration/DurableMessagingOptions.cs | 5 + src/Orleans.Messaging/Envelope.cs | 3 +- src/Orleans.Messaging/IOutbox.cs | 4 + src/Orleans.Messaging/Inbox.cs | 42 +++ src/Orleans.Messaging/MessagingPump.cs | 2 +- src/Orleans.Messaging/Outbox.cs | 27 +- src/Orleans.Messaging/README.md | 25 +- .../Functional/JsonJournalMessagingTests.cs | 41 ++- .../Functional/ScheduledDeliveryTests.cs | 4 +- .../Functional/SelfAddressedMessageTests.cs | 327 ++++++++++++++++++ 10 files changed, 460 insertions(+), 20 deletions(-) create mode 100644 test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index 37982d12a54..cda885888d6 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -47,6 +47,11 @@ public class DurableMessagingOptions /// Messages with an time do not count toward this limit, including after they /// become due, and are accepted regardless of it. /// + /// + /// Messages a grain sends to itself are accepted regardless of this limit, because sending is synchronous. + /// Those without an time count toward it until they complete, so they can cause + /// deliveries from other grains to be backpressured. + /// /// /// /// The maximum inbox capacity. Must be greater than zero. Defaults to 1000. diff --git a/src/Orleans.Messaging/Envelope.cs b/src/Orleans.Messaging/Envelope.cs index 8393beee452..ef9ec55c849 100644 --- a/src/Orleans.Messaging/Envelope.cs +++ b/src/Orleans.Messaging/Envelope.cs @@ -36,7 +36,8 @@ public readonly struct Envelope /// /// The outbox delivers the message immediately. The receiving inbox accepts, deduplicates, and persists it /// as usual, then holds it until this time is reached on the Durable Jobs clock, and one durable wake-up - /// resumes the grain when the message becomes due. processes the message as soon as + /// resumes the grain when the message becomes due. A message a grain sends to itself is accepted by its own + /// inbox in the sending journal write. processes the message as soon as /// its acceptance is durable. /// /// diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index b9f74136b54..18b8424b4d1 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -27,6 +27,10 @@ public interface IOutbox /// or . /// Ordinary callers persist staged messages using their journaled state manager. /// An explicit write retry retains pending business changes and messages after a scheduling failure. + /// A message addressed to the sending grain bypasses the outbox: the grain's own inbox stages its acceptance + /// for the same journal write, applying the inbox identity rules, so a completed ID is a duplicate and a different + /// pending command with the ID conflicts. The inbox admits such a message regardless of its capacity and of handler + /// registration. /// /// The envelope has an unset identity, invalid subject or destination, or exceeds identity/subject limits. /// diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index c856d43aa7d..cb6cef28a03 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -133,6 +133,48 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel } } + /// + /// Stages acceptance of a message which this grain sends to itself, for the caller's next journal write. + /// + /// + /// + /// Applies the identity rules of synchronously and without writing: a completed or + /// equivalent pending message is a duplicate, and a different pending command with the same ID conflicts. + /// The acceptance becomes durable atomically with the caller's other staged state, after which the inbox processes + /// the message as usual. When the pump is not running yet, the durable wake-up which the write ensures for new + /// acceptances starts it once the activation services calls, so handlers never run during activation. + /// + /// + /// A synchronous send cannot report backpressure or a missing handler, so the message is admitted regardless of + /// and of handler registration. A message without + /// still occupies capacity for deliveries from other grains until it completes, + /// and processing waits for a handler to be registered. + /// + /// + /// The inbox holds a different pending command with the same ID. + internal void StageLocal(InboxMessage message) + { + if (IsDuplicate(message, out _)) + { + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "duplicate"); + return; + } + + try + { + Emit(new() { Kind = InboxCommandKind.Accept, Message = message }); + } + catch (Exception exception) + { + LogStagingFailed(Logger, exception); + _pump.Fail(exception, "Inbox staging failed."); + throw; + } + + Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, "accepted"); + LogMessageAccepted(Logger, message.Envelope.MessageId, GrainContext.GrainId); + } + /// /// Offers a message to the registered sinks and stages the outcome which the first sink to take the message reports. /// diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 395c1b627d9..9daee586482 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -76,7 +76,7 @@ public MessagingPump(IGrainContext grainContext, IJournaledStateManager stateMan internal void Register(Outbox outbox) => _outbox = outbox; - private Inbox Inbox => _inbox ?? throw new InvalidOperationException("The messaging inbox has not been registered."); + internal Inbox Inbox => _inbox ?? throw new InvalidOperationException("The messaging inbox has not been registered."); private Outbox Outbox => _outbox ?? throw new InvalidOperationException("The messaging outbox has not been registered."); diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index b76913b2a20..e3d34d52169 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -56,6 +56,14 @@ public void Send(OutboxMessage message) return; } + if (message.ReceiverId == GrainId) + { + // The grain's own inbox accepts the message in the caller's journal write, so the outbox never holds it. + _pump.Inbox.StageLocal(message.ToInboxMessage()); + Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); + return; + } + try { Emit(new() { Kind = OutboxCommandKind.Enqueue, Message = message, State = new() { EnqueuedAt = TimeProvider.GetUtcNow() } }); @@ -81,6 +89,10 @@ private void OnRecoveryCompleted() { throw new InvalidOperationException("The recovered outbox key does not match its command identity."); } + if (message.ReceiverId == GrainId) + { + throw new InvalidOperationException("A recovered outbox message is addressed to its own grain, whose inbox accepts such messages directly."); + } } foreach (var (key, letter) in _deadLetters) { @@ -89,6 +101,10 @@ private void OnRecoveryCompleted() { throw new InvalidOperationException("The recovered outbox dead-letter key does not match its message identity."); } + if (letter.Message.ReceiverId == GrainId) + { + throw new InvalidOperationException("A recovered outbox dead letter is addressed to its own grain, whose inbox accepts such messages directly."); + } } _capturedSequence = _durableSequence = _lastSequence; } @@ -154,9 +170,7 @@ private async Task DeliverAsync(OutboxEntry entry, Cancellation var message = entry.Message; try { - var target = message.ReceiverId == GrainId - ? _pump.GrainContext.GetGrainExtension() - : Shared.GrainFactory.GetGrain(message.ReceiverId); + var target = Shared.GrainFactory.GetGrain(message.ReceiverId); var result = await target.DeliverAsync(message.ToInboxMessage(), cancellationToken).ConfigureAwait(true); return new(entry, result, null, Stopwatch.GetElapsedTime(start), false); } @@ -191,12 +205,7 @@ internal async ValueTask AdvanceAsync(CancellationToken cancellationToken) var attempts = new Task[candidates.Count]; for (var index = 0; index < attempts.Length; index++) { - var entry = candidates[index]; - attempts[index] = DeliverAsync(entry, _pump.ShutdownToken); - if (entry.Message.ReceiverId == GrainId) - { - await attempts[index].ConfigureAwait(true); - } + attempts[index] = DeliverAsync(candidates[index], _pump.ShutdownToken); } var deliveries = Task.WhenAll(attempts); diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 6ff719b607b..3ce8a87b33e 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -4,7 +4,8 @@ built on Orleans Journaling and Durable Jobs. A grain sends a message by staging it in its outbox; the message persists with the grain's other journaled state and is delivered to the receiver's inbox, which processes it at least once and deduplicates -it by message ID. +it by message ID. A message a grain sends to itself skips the outbox: its own inbox +accepts it in the same journal write. ## Setup @@ -90,6 +91,17 @@ then apply state changes, send messages, and complete in one synchronous block. inbox delivery. Sending equivalent content with a pending ID coalesces with the original message; different content throws `InvalidOperationException`. +When the receiver is the sending grain, `Send` stages the message's acceptance in +the grain's own inbox instead of the outbox, applying the inbox's identity rules: an +ID that is pending with equivalent content or completed within `DeduplicationWindow` +is a duplicate, and a pending ID with different content throws +`InvalidOperationException`. The acceptance persists in the caller's next journal +write, atomically with its other staged state, and the inbox processes the message +once that write completes. Because `Send` is synchronous, the grain's inbox admits the +message regardless of `MaxCapacity` and of handler registration; an unscheduled +self-sent message still occupies capacity for other senders until it completes, and +processing waits for a handler. + ## Journal payloads once When a payload already lives in the grain's own journaled state, an outbox source or @@ -143,7 +155,8 @@ write completes. Each receiver returns a `DeliveryResult`: A completed message which an inbox sink rejected is answered with its original `DeadLettered` result instead. - `Backpressured`: the inbox holds `MaxCapacity` pending messages without `DeliverAt`. - The inbox admits scheduled messages regardless of `MaxCapacity`. + The inbox admits scheduled messages and messages its grain sends to itself + regardless of `MaxCapacity`. - `HandlerNotFound`: the receiver has no registered handlers. - `DeadLettered`: the receiver moved the message to its dead-letter store. @@ -178,6 +191,8 @@ grain's next journal write. The `Microsoft.Orleans` meter publishes counters for received, processed, sent, and delivered messages tagged by grain type and status; histograms for processing and delivery duration; and `orleans-messaging-inbox-depth` and -`orleans-messaging-outbox-depth` up-down counters for pending work. A message an inbox -sink accepts counts as received `accepted` and processed `success`; one which the -sink dead-letters counts as received and processed `dead_lettered`. +`orleans-messaging-outbox-depth` up-down counters for pending work. A message a grain +sends to itself counts as sent and as received by its inbox, and never contributes +to outbox delivery or depth. A message an inbox sink accepts counts as received +`accepted` and processed `success`; one which the sink dead-letters counts as received +and processed `dead_lettered`. diff --git a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs index 4e936f98f3c..494e42567d6 100644 --- a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs @@ -89,6 +89,37 @@ public async Task JsonJournal_RecoversScheduledInboxMessageWithItsDeliverAt() Assert.Empty(Probe.Get(receiver.GetGrainId())); } + [Fact] + public async Task JsonJournal_RecoversSelfSentMessagesFromTheInbox() + { + var grain = Cluster.Client.GetGrain(Guid.NewGuid()); + var scheduledId = HierarchicalKey.Create("json", "self-scheduled", Guid.NewGuid().ToString("N")); + var immediateId = HierarchicalKey.Create("json", "self", Guid.NewGuid().ToString("N")); + var deliverAt = new DateTimeOffset(DateTime.UtcNow.AddDays(1).Ticks, TimeSpan.Zero); + + // Each send commits the acceptance in the grain's own inbox; the outbox never holds the message. + await grain.SendAsync(grain.GetGrainId(), scheduledId, 7, deliverAt); + await grain.SendAsync(grain.GetGrainId(), immediateId, 8); + Assert.Equal(0, await grain.GetOutboxCountAsync()); + Assert.Equal(new[] { 8 }, await Probe.WaitAsync(grain.GetGrainId(), 1, Token)); + Assert.True(Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)); + await grain.DeactivateAsync(); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + + // The next activation recovers the pending scheduled message and the completion record from the JSON journal. + Assert.Equal(0, await grain.GetOutboxCountAsync()); + Assert.True(Cluster.TryGetGrainContext(grain.GetGrainId(), out var recoveredContext)); + Assert.NotSame(context, recoveredContext); + var inbox = recoveredContext.ActivationServices.GetRequiredService(); + var recovered = new TaskCompletionSource<(int Count, DateTimeOffset? DeliverAt)>(TaskCreationOptions.RunContinuationsAsynchronously); + recoveredContext.Scheduler.QueueAction(() => recovered.SetResult((inbox.Count, inbox.MessageState[scheduledId].Message.Envelope.DeliverAt))); + Assert.Equal((1, deliverAt), await recovered.Task.WaitAsync(TimeSpan.FromSeconds(30), Token)); + var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); + var redelivery = type.Create(immediateId, grain.GetGrainId(), 8).ToInboxMessage(); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(redelivery, Token)).Status); + Assert.Equal(new[] { 8 }, Probe.Get(grain.GetGrainId())); + } + public sealed class Fixture : IAsyncLifetime { public JsonMessagingProbe Probe { get; } = new(); @@ -140,6 +171,7 @@ public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken can } public interface IJsonMessagingGrain : IGrainWithGuidKey { + Task SendAsync(GrainId destination, HierarchicalKey messageId, int value, DateTimeOffset? deliverAt = null); Task SendAndDeactivateAsync(GrainId destination, HierarchicalKey messageId, int value); Task GetOutboxCountAsync(); Task DeactivateAsync(); @@ -187,10 +219,15 @@ public JsonMessagingGrain(IJournaledStateManager manager, IInbox inbox, IOutbox }); } - public async Task SendAndDeactivateAsync(GrainId destination, HierarchicalKey messageId, int value) + public async Task SendAsync(GrainId destination, HierarchicalKey messageId, int value, DateTimeOffset? deliverAt = null) { - _outbox.Send(_type, messageId, destination, value); + _outbox.Send(_type, messageId, destination, value, deliverAt); await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task SendAndDeactivateAsync(GrainId destination, HierarchicalKey messageId, int value) + { + await SendAsync(destination, messageId, value); DeactivateOnIdle(); } diff --git a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs index 507db0e4920..2ae0d06a749 100644 --- a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs @@ -201,7 +201,7 @@ public async Task ScheduledMessage_RedeliveryCoalescesAndDifferentDeliverAtConfl } [Fact] - public async Task SelfSentScheduledMessage_IsDeliveredImmediatelyAndProcessedWhenDue() + public async Task SelfSentScheduledMessage_IsAcceptedWithItsWriteAndProcessedWhenDue() { var grain = NewGrain(); _ = await grain.GetSnapshotAsync(); @@ -213,7 +213,7 @@ public async Task SelfSentScheduledMessage_IsDeliveredImmediatelyAndProcessedWhe await grain.StageOutputAsync(outgoing); await grain.RetryWriteStateAsync(); - // The outbox hands the message to its own inbox without waiting for the due time. + // The grain's own inbox accepted the message in that write, without waiting for the due time. var delivered = await Fixture.SnapshotProbe.WaitAsync( grain.GetGrainId(), static snapshot => snapshot.InboxCount == 1 && snapshot.OutboxCount == 0); Assert.Empty(delivered.Effects); diff --git a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs new file mode 100644 index 00000000000..e6f1fcecdf7 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs @@ -0,0 +1,327 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Messages a grain sends to itself are accepted by its own inbox in the caller's journal write, without an outbox entry. +/// +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class SelfAddressedMessageTests() : MessagingBehaviorTestBase(new SelfAddressedClusterFixture()) +{ + private static readonly TimeSpan Delay = TimeSpan.FromHours(1); + + private ManualJobClock JobClock => ((SelfAddressedClusterFixture)Fixture).JobClock; + + [Fact] + public async Task SelfSend_IsAcceptedWithoutOutboxEntryInTheCallersSingleWrite() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var instance = GetInstance(grain); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + const string route = "messages/self-single-write"; + using var held = Fixture.HandlerProbe.Arm(grain.GetGrainId(), route); + var writes = Fixture.Storage.GetSuccessfulWriteCount(journal); + var effect = new MessageEffect(TestApplicationProtocol.NewMessageId(), 1, 0, "caller-state"); + + await grain.StageEffectAsync(effect); + await grain.StageOutputAsync(SelfMessage(grain, NewMessage(1, "self"), route)); + var staged = await grain.GetSnapshotAsync(); + Assert.Equal(1, staged.InboxCount); + Assert.Equal(0, staged.OutboxCount); + Assert.Equal(writes, Fixture.Storage.GetSuccessfulWriteCount(journal)); + + instance.ClearCaptures(); + await grain.RetryWriteStateAsync(); + var written = Assert.Single(instance.Captures); + Assert.Equal(1, written.InboxCount); + Assert.Equal(0, written.OutboxCount); + Assert.Equal(effect, Assert.Single(written.Effects)); + Assert.Empty(instance.OutputCaptures[^1]); + + // Processing starts after that one write; the handler is held before it stages any further write. + await WaitForBarrierAsync(grain, held); + Assert.Equal(writes + 1, Fixture.Storage.GetSuccessfulWriteCount(journal)); + held.Release(); + var completed = await Fixture.WaitForEffectCountAsync(grain, 2); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(1, completed.ProcessedMessageCount); + Assert.Equal(writes + 2, Fixture.Storage.GetSuccessfulWriteCount(journal)); + } + + [Fact] + public async Task SelfSend_IsProcessedOnlyAfterItsWriteIsDurable() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var instance = GetInstance(grain); + var storage = Fixture.Storage.BlockWrite(JournalId.FromGrainId(grain.GetGrainId())); + + var send = grain.SendAsync(grain.GetGrainId(), "messages/self-blocked", NewMessage(2, "blocked")); + await storage.WaitUntilEnteredAsync(); + + // A full pump pass while the write is in flight leaves the accepted message pending. + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await Fixture.RunPumpAsync(grain)).Status); + var pending = await Fixture.OnGrainAsync(grain, () => instance.GetSnapshotForTest()); + Assert.Equal(1, pending.InboxCount); + Assert.Equal(0, pending.OutboxCount); + Assert.Empty(pending.Effects); + + storage.Release(); + await send; + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal("blocked", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + } + + [Fact] + public async Task SelfSend_IsDiscardedWhenItsWriteFails() + { + var grain = NewGrain(); + var initial = await grain.GetSnapshotAsync(); + var context = Fixture.GetGrainContext(grain); + Fixture.Storage.FailWrite(JournalId.FromGrainId(grain.GetGrainId())); + + await Assert.ThrowsAnyAsync(() => grain.SendAsync(grain.GetGrainId(), "messages/self-failed", NewMessage(3, "failed"))); + + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await grain.GetSnapshotAsync(); + Assert.NotEqual(initial.ActivationId, recovered.ActivationId); + Assert.Equal(0, recovered.InboxCount); + Assert.Equal(0, recovered.OutboxCount); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var after = await grain.GetSnapshotAsync(); + Assert.Empty(after.Effects); + Assert.Equal(0, after.ProcessedMessageCount); + } + + [Fact] + public async Task SelfSend_FollowsInboxIdentityRules() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + const string route = "messages/self-identity"; + var message = SelfMessage(grain, NewMessage(4, "identity"), route); + var differentBody = SelfMessage(grain, NewMessage(5, "different"), route, messageId: message.Envelope.MessageId); + var differentDeliverAt = WithDeliverAt(message, JobClock.GetUtcNow() + Delay); + + using (var held = Fixture.HandlerProbe.Arm(grain.GetGrainId(), route)) + { + await grain.StageOutputAsync(message); + await grain.StageOutputAsync(message); + await Assert.ThrowsAsync(() => grain.StageOutputAsync(differentBody)); + await Assert.ThrowsAsync(() => grain.StageOutputAsync(differentDeliverAt)); + Assert.Equal(1, (await grain.GetSnapshotAsync()).InboxCount); + + // The outbox keys pending messages by ID, so the ID cannot address another grain while it is pending there. + var other = SelfMessage(grain, NewMessage(6, "other"), route); + var otherReceiver = NewGrain().GetGrainId(); + await grain.StageOutputAsync(new OutboxMessage { ReceiverId = otherReceiver, Envelope = other.Envelope }); + await Assert.ThrowsAsync(() => grain.StageOutputAsync(other)); + + await grain.RetryWriteStateAsync(); + await WaitForBarrierAsync(grain, held); + } + + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal(1, completed.ProcessedMessageCount); + + // A completed ID is a duplicate regardless of content while its completion record is retained. + await grain.StageOutputAsync(message); + await grain.StageOutputAsync(differentBody); + var repeated = await grain.GetSnapshotAsync(); + Assert.Equal(0, repeated.InboxCount); + await grain.RetryWriteStateAsync(); + _ = await Fixture.RunPumpAsync(grain); + var final = await grain.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(final.Effects).Count); + Assert.Empty(final.InboxDeadLetters); + } + + [Fact] + public async Task SelfSend_IsAdmittedBeyondCapacityAndWithoutHandlers() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + await grain.ConfigureHandlerAsync(false); + await ReactivateAsync(grain); + + // MaxCapacity is 2 and no handler is registered; a synchronous send admits every message. + for (var sequence = 10; sequence < 13; sequence++) + { + await grain.StageOutputAsync(SelfMessage(grain, NewMessage(sequence, $"self-{sequence}"), "messages/self-capacity")); + } + + await grain.RetryWriteStateAsync(); + var admitted = await grain.GetSnapshotAsync(); + Assert.Equal(3, admitted.InboxCount); + Assert.Equal(0, admitted.OutboxCount); + + // Unscheduled self-sent messages occupy capacity for deliveries from other grains. + var external = CreateEnvelope(grain, NewMessage(20, "external")); + Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(grain, external)).Status); + Assert.NotEqual(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var waiting = await grain.GetSnapshotAsync(); + Assert.Equal(3, waiting.InboxCount); + Assert.Empty(waiting.Effects); + Assert.Empty(waiting.InboxDeadLetters); + + await grain.ConfigureHandlerAsync(true); + await ReactivateAsync(grain); + _ = await Fixture.RunPumpAsync(grain); + var completed = await Fixture.WaitForEffectCountAsync(grain, 3); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(grain, external)).Status); + await Fixture.WaitForEffectCountAsync(grain, 4); + } + + [Fact] + public async Task SelfSendFromHandler_IsProcessedInALaterBatch() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var instance = GetInstance(grain); + instance.ClearCaptures(); + var message = NewMessage(30, "forwarding") with { ForwardTo = grain.GetGrainId() }; + + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(grain, CreateEnvelope(grain, message))).Status); + + var completed = await Fixture.WaitForEffectCountAsync(grain, 2); + Assert.Equal(2, Assert.Single(completed.Effects).Count); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + Assert.Equal(2, completed.ProcessedMessageCount); + + // The batch which processed the original also accepted the forwarded message, which a later batch processed. + var captures = instance.Captures; + var original = Assert.Single(captures, static capture => capture.Effects.Sum(static effect => effect.Count) == 1); + Assert.Equal(1, original.InboxCount); + Assert.Equal(0, original.OutboxCount); + Assert.Equal(1, original.ProcessedMessageCount); + Assert.All(instance.OutputCaptures, static output => Assert.Empty(output)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ScheduledSelfSend_SurvivesRecoveryAndIsProcessedWhenDue(bool snapshot) + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(grain.GetGrainId()); + if (snapshot) + { + Fixture.Storage.RequestSnapshot(journal); + } + + var deliverAt = JobClock.GetUtcNow() + Delay; + var message = WithDeliverAt(SelfMessage(grain, NewMessage(40, "recovered"), "messages/self-recovered"), deliverAt); + await grain.StageOutputAsync(message); + await grain.RetryWriteStateAsync(); + + await ReactivateAsync(grain); + var recovered = await grain.GetSnapshotAsync(); + Assert.Equal(1, recovered.InboxCount); + Assert.Equal(0, recovered.OutboxCount); + Assert.Empty(recovered.Effects); + var inbox = Fixture.GetInbox(grain); + Assert.Equal(deliverAt, await Fixture.OnGrainAsync(grain, + () => InboxTestAccess.GetPendingMessages(inbox)[message.Envelope.MessageId].Message.Envelope.DeliverAt)); + + var held = await Fixture.RunPumpAsync(grain); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, held.Status); + Assert.Equal(deliverAt, held.RescheduleTime); + + // The recovered pending message deduplicates a repeated send. + await grain.StageOutputAsync(message); + Assert.Equal(1, (await grain.GetSnapshotAsync()).InboxCount); + + JobClock.Advance(Delay); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + var completed = await Fixture.WaitForEffectCountAsync(grain, 1); + Assert.Equal("recovered", Assert.Single(completed.Effects).Value); + Assert.Equal(0, completed.InboxCount); + Assert.Equal(0, completed.OutboxCount); + } + + [Fact] + public async Task SelfSend_RecordsSendAndReceiptWithoutOutboxMetrics() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var services = Fixture.GetGrainContext(grain).ActivationServices; + using var probe = new MessageMetricListener(services.GetRequiredService().Meter); + var grainType = grain.GetGrainId().Type.ToString(); + const string route = "messages/self-metrics"; + var message = SelfMessage(grain, NewMessage(50, "metrics"), route); + + using (var held = Fixture.HandlerProbe.Arm(grain.GetGrainId(), route)) + { + await grain.StageOutputAsync(message); + await grain.StageOutputAsync(message); + await grain.RetryWriteStateAsync(); + await WaitForBarrierAsync(grain, held); + } + + await Fixture.WaitForEffectCountAsync(grain, 1); + var sent = probe.Read("outbox-messages-sent"); + Assert.Equal(2, sent.Length); + Assert.All(sent, row => row.AssertCounter(grainType)); + var received = probe.Read("inbox-messages-received"); + Assert.Equal(2, received.Length); + received[0].AssertCounter(grainType, "accepted"); + received[1].AssertCounter(grainType, "duplicate"); + Assert.Equal([1d, -1d], probe.Read("inbox-depth").Select(static row => row.Value)); + Assert.Empty(probe.Read("outbox-depth")); + Assert.Empty(probe.Read("outbox-messages-delivered")); + Assert.Empty(probe.Read("outbox-delivery-duration")); + } + + [Fact] + public async Task RecoveredOutboxEntryAddressedToItsOwnGrain_IsRejected() + { + var grain = NewGrain(); + _ = await grain.GetSnapshotAsync(); + var outbox = Fixture.GetGrainContext(grain).ActivationServices.GetRequiredService(); + var message = SelfMessage(grain, NewMessage(60, "outbox-entry"), "messages/self-outbox-entry"); + + // Replays an outbox entry which a correct journal never contains, then validates the recovered state. + var error = await Assert.ThrowsAsync(() => Fixture.OnGrainAsync(grain, () => + { + ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand + { + Kind = OutboxCommandKind.Enqueue, + Message = message, + State = new() { EnqueuedAt = Fixture.Clock.GetUtcNow() } + }); + ((IStateMachine)outbox).OnRecoveryCompleted(); + })); + Assert.Contains("addressed to its own grain", error.Message, StringComparison.Ordinal); + } + + private MessagingTestGrain GetInstance(IMessagingTestGrain grain) => + Assert.IsType(Fixture.GetGrainContext(grain).GrainInstance); + + private OutboxMessage SelfMessage(IMessagingTestGrain grain, TestMessage message, string route, HierarchicalKey messageId = default) => + TestApplicationProtocol.Create(Sessions, grain.GetGrainId(), grain.GetGrainId(), route, message, messageId); + + private static OutboxMessage WithDeliverAt(OutboxMessage message, DateTimeOffset? deliverAt) => + new() { ReceiverId = message.ReceiverId, Envelope = message.Envelope with { DeliverAt = deliverAt } }; + + private sealed class SelfAddressedClusterFixture() : MessagingClusterFixture(1, receiverOnly: false) + { + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); + + protected override void ConfigureServices(IServiceCollection services) => + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + } +} From f8191e62af03baa7d60abc72f12e3ecf7afaa702 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 11:18:31 -0700 Subject: [PATCH 70/95] feat(messaging): journal outbox messages by reference to application-owned sources IOutbox.Send(HierarchicalKey) stages a reference entry which journals only the message ID and delivery state. The first registered IOutboxMessageSource whose TryGetMessage returns the message owns it; the owner is cached in memory only. Each delivery pass asks the owner for the message, and a false return withdraws it: the entry is removed without delivery or OnRetired and recorded with the 'withdrawn' status. Retirement calls OnRetired while staging the removal, so the source's changes commit in the same journal write. Recovery fails only when reference entries exist and no source is registered. Self-sent messages are offered to inbox sinks. --- src/Orleans.Messaging/Inbox.cs | 5 + src/Orleans.Messaging/Outbox.State.cs | 38 +- src/Orleans.Messaging/Outbox.cs | 204 +++++++++-- src/Orleans.Messaging/README.md | 28 +- .../MessagingCommandSerializationTests.cs | 28 +- .../Functional/MessageSourceAndSinkTests.cs | 324 +++++++++++++++++- .../Support/InboxTestAccess.cs | 2 +- 7 files changed, 587 insertions(+), 42 deletions(-) diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index cb6cef28a03..9cbc0383582 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -160,6 +160,11 @@ internal void StageLocal(InboxMessage message) return; } + if (message.Envelope.DeliverAt is null && StageSinkAcceptance(message) is not null) + { + return; + } + try { Emit(new() { Kind = InboxCommandKind.Accept, Message = message }); diff --git a/src/Orleans.Messaging/Outbox.State.cs b/src/Orleans.Messaging/Outbox.State.cs index 1c0a048831b..4647933c4b3 100644 --- a/src/Orleans.Messaging/Outbox.State.cs +++ b/src/Orleans.Messaging/Outbox.State.cs @@ -67,9 +67,9 @@ private void Apply(OutboxCommand command) switch (command.Kind) { case OutboxCommandKind.Enqueue: - var message = command.Message ?? throw new InvalidOperationException("An enqueue command requires its message."); var state = command.State ?? throw new InvalidOperationException("An enqueue command requires its delivery state."); - _messages.Add(message.Envelope.MessageId, new OutboxEntry { Message = message, State = state, Sequence = ++_lastSequence }); + var entry = new OutboxEntry { Message = command.Message, Reference = command.Reference, State = state, Sequence = ++_lastSequence }; + _messages.Add(entry.MessageId, entry); ReportDepth(1); break; case OutboxCommandKind.Delivery: @@ -89,7 +89,7 @@ private void Apply(OutboxCommand command) var previousCount = _messages.Count; _messages.Clear(); _deadLetters.Clear(); - foreach (var restored in snapshot.Messages) _messages.Add(restored.Message.Envelope.MessageId, restored); + foreach (var restored in snapshot.Messages) _messages.Add(restored.MessageId, restored); foreach (var restored in snapshot.DeadLetters) _deadLetters.Add(restored.Message.Envelope.MessageId, restored); ReportDepth(_messages.Count - previousCount); break; @@ -125,6 +125,18 @@ internal sealed class OutboxCommand [Id(4)] public OutboxDeadLetter? DeadLetter { get; init; } [Id(5)] public HierarchicalKey[]? RemovedDeadLetters { get; init; } [Id(6)] public OutboxSnapshot? Snapshot { get; init; } + + // An enqueue command sets this instead of Message for a message which an IOutboxMessageSource provides. + [Id(7)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OutboxMessageReference? Reference { get; init; } +} + +/// Identifies a message which an provides. +[GenerateSerializer, Alias("Orleans.Messaging.OutboxMessageReference")] +internal readonly struct OutboxMessageReference +{ + [Id(0)] public required HierarchicalKey MessageId { get; init; } } [GenerateSerializer, Alias("Orleans.Messaging.OutboxSnapshot")] @@ -137,10 +149,28 @@ internal sealed class OutboxSnapshot [GenerateSerializer, Alias("Orleans.Messaging.OutboxEntry")] internal sealed class OutboxEntry { - [Id(0)] public required OutboxMessage Message { get; init; } + // Exactly one of Message and Reference is set. A reference entry journals no payload or receiver: its source provides the message. + [Id(0)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OutboxMessage? Message { get; init; } + [Id(1)] public required OutboxMessageState State { get; set; } + + [Id(2)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OutboxMessageReference? Reference { get; init; } + [field: NonSerialized] [JsonIgnore] public long Sequence { get; init; } + + /// Gets the source which owns a reference entry's message, resolved when it is sent or first materialized after recovery. + [field: NonSerialized] + [JsonIgnore] public IOutboxMessageSource? Source { get; set; } + + [JsonIgnore] + public HierarchicalKey MessageId => Message is { } message ? message.Envelope.MessageId + : Reference is { } reference ? reference.MessageId + : throw new InvalidOperationException("An outbox entry requires its message or message reference."); } [JsonSerializable(typeof(OutboxCommand))] diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index e3d34d52169..65e522ab6f8 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -22,6 +22,7 @@ internal sealed partial class Outbox : IStateMachine, IDurableValueCommandHandle private JournalStreamWriter _writer; private readonly Dictionary _messages = []; private readonly Dictionary _deadLetters = []; + private readonly List _sources = []; private Task? _deliveries; public Outbox(MessagingPump pump) @@ -42,13 +43,20 @@ public Outbox(MessagingPump pump) internal IReadOnlyDictionary DeadLetterState => _deadLetters; internal int Count => _messages.Count; + public IOutbox Register(IOutboxMessageSource source) + { + ArgumentNullException.ThrowIfNull(source); + _sources.Add(source); + return this; + } + public void Send(OutboxMessage message) { _pump.ShutdownToken.ThrowIfCancellationRequested(); EnvelopeValidation.Validate(message); if (_messages.TryGetValue(message.Envelope.MessageId, out var existing)) { - if (!EnvelopeEquivalence.AreEquivalent(existing.Message, message)) + if (existing.Message is not { } pending || !EnvelopeEquivalence.AreEquivalent(pending, message)) { throw new InvalidOperationException( $"The outbox already contains a different envelope with message ID '{message.Envelope.MessageId}'."); @@ -75,21 +83,131 @@ public void Send(OutboxMessage message) } } + public void Send(HierarchicalKey messageId) + { + _pump.ShutdownToken.ThrowIfCancellationRequested(); + EnvelopeValidation.ValidateMessageId(messageId); + if (_messages.TryGetValue(messageId, out var existing)) + { + // The source is authoritative for a pending reference entry's message, so a repeated send coalesces. + if (existing.Reference is null) + { + throw new InvalidOperationException( + $"The outbox already contains a different message with message ID '{messageId}'."); + } + return; + } + + if (FindSource(messageId, out _) is not { } source) + { + throw new InvalidOperationException($"No registered outbox message source provides message '{messageId}'."); + } + + try + { + Emit(new() + { + Kind = OutboxCommandKind.Enqueue, + Reference = new() { MessageId = messageId }, + State = new() { EnqueuedAt = TimeProvider.GetUtcNow() } + }); + _messages[messageId].Source = source; + Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); + } + catch (Exception exception) + { + FailStaging(exception); + } + } + + /// Returns the first registered source which provides the message, and its validated message. + private IOutboxMessageSource? FindSource(HierarchicalKey messageId, out OutboxMessage message) + { + foreach (var source in _sources) + { + if (TryGetSourceMessage(source, messageId, out message)) return source; + } + + message = default; + return null; + } + + /// + /// Returns the entry's message, materializing a reference entry's message from its owning source, or returns + /// when the source has withdrawn the message. + /// + private bool TryGetMessage(OutboxEntry entry, out OutboxMessage message) + { + if (entry.Message is { } ordinary) + { + message = ordinary; + return true; + } + + if (entry.Source is { } owner) return TryGetSourceMessage(owner, entry.MessageId, out message); + entry.Source = FindSource(entry.MessageId, out message); + return entry.Source is not null; + } + + private bool TryGetSourceMessage(IOutboxMessageSource source, HierarchicalKey messageId, out OutboxMessage message) + { + if (!source.TryGetMessage(messageId, out message)) return false; + EnvelopeValidation.Validate(message); + if (message.Envelope.MessageId != messageId) + { + throw new InvalidOperationException( + $"The outbox message source returned message ID '{message.Envelope.MessageId}' for message '{messageId}'."); + } + if (message.ReceiverId == GrainId) + { + // The grain's own inbox would hold the payload, so a source cannot keep it as the single copy. + throw new ArgumentException( + $"The outbox message source addressed message '{messageId}' to the sending grain; send its message with Send(OutboxMessage) instead.", + nameof(messageId)); + } + + return true; + } + + /// Notifies the source of a retired reference entry, so its release commits with the entry's removal. + private static void Retire(OutboxEntry entry, OutboxMessageRetirement retirement) + { + if (entry.Reference is not null) entry.Source!.OnRetired(entry.MessageId, retirement); + } + private void OnRecoveryCompleted() { + if (_sources.Count == 0 && _messages.Values.Any(static entry => entry.Reference is not null)) + { + throw new InvalidOperationException( + "The outbox recovered messages which a source provides, but no outbox message source is registered. Register outbox message sources during activation setup, before journal recovery."); + } + foreach (var (key, entry) in _messages) { - var message = entry.Message; - EnvelopeValidation.Validate(message); + if (entry.Message is { } message) + { + if (entry.Reference is not null) + { + throw new InvalidOperationException("A recovered outbox entry has both a message and a message reference."); + } + + EnvelopeValidation.Validate(message); + } + else + { + EnvelopeValidation.ValidateMessageId(key); + } + if (entry.State.EnqueuedAt is null) { throw new InvalidOperationException("A recovered outbox message requires its enqueue time."); } - if (key != message.Envelope.MessageId) + if (key != entry.MessageId) { throw new InvalidOperationException("The recovered outbox key does not match its command identity."); } - if (message.ReceiverId == GrainId) + if (entry.Message?.ReceiverId == GrainId) { throw new InvalidOperationException("A recovered outbox message is addressed to its own grain, whose inbox accepts such messages directly."); } @@ -138,15 +256,22 @@ private async ValueTask PersistAsync() } } - private List? SelectMessages() + private List<(OutboxEntry Entry, OutboxMessage Message)>? SelectMessages(ref List? withdrawn) { var now = TimeProvider.GetUtcNow(); - List? candidates = null; + List<(OutboxEntry Entry, OutboxMessage Message)>? candidates = null; foreach (var entry in _messages.Values) { if (entry.Sequence <= _durableSequence && IsReadyForAttempt(entry.State, now)) { - (candidates ??= []).Add(entry); + // An invalid source message fails the pass before any attempt starts. + if (!TryGetMessage(entry, out var message)) + { + (withdrawn ??= []).Add(entry); + continue; + } + + (candidates ??= []).Add((entry, message)); if (candidates.Count == Options.OutboxBatchSize) break; } } @@ -159,24 +284,23 @@ private async ValueTask PersistAsync() return candidates; } - private async Task DeliverAsync(OutboxEntry entry, CancellationToken cancellationToken) + private async Task DeliverAsync(OutboxEntry entry, OutboxMessage message, CancellationToken cancellationToken) { if (entry.State.EnqueuedAt is { } enqueuedAt && MessagingTime.IsExpired(TimeProvider.GetUtcNow(), enqueuedAt, Options.MaxOutboxRetryAge)) { - return new(entry, null, $"The message exceeded the maximum retry age of {Options.MaxOutboxRetryAge}.", TimeSpan.Zero, true); + return new(entry, message, null, $"The message exceeded the maximum retry age of {Options.MaxOutboxRetryAge}.", TimeSpan.Zero, true); } var start = Stopwatch.GetTimestamp(); - var message = entry.Message; try { var target = Shared.GrainFactory.GetGrain(message.ReceiverId); var result = await target.DeliverAsync(message.ToInboxMessage(), cancellationToken).ConfigureAwait(true); - return new(entry, result, null, Stopwatch.GetElapsedTime(start), false); + return new(entry, message, result, null, Stopwatch.GetElapsedTime(start), false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - return new(entry, null, "Delivery was canceled.", Stopwatch.GetElapsedTime(start), false); + return new(entry, message, null, "Delivery was canceled.", Stopwatch.GetElapsedTime(start), false); } catch (Exception exception) { @@ -184,7 +308,7 @@ private async Task DeliverAsync(OutboxEntry entry, Cancellation { LogDeliveryError(Logger, exception, message.Envelope.MessageId, message.ReceiverId); } - return new(entry, null, exception.ToString(), Stopwatch.GetElapsedTime(start), false); + return new(entry, message, null, exception.ToString(), Stopwatch.GetElapsedTime(start), false); } } @@ -200,12 +324,15 @@ internal async ValueTask AdvanceAsync(CancellationToken cancellationToken) return true; } - var candidates = SelectMessages(); - if (candidates is null) return false; + List? withdrawn = null; + var candidates = SelectMessages(ref withdrawn); + if (withdrawn is not null) await RemoveWithdrawnAsync(withdrawn).ConfigureAwait(true); + if (candidates is null) return withdrawn is not null; var attempts = new Task[candidates.Count]; for (var index = 0; index < attempts.Length; index++) { - attempts[index] = DeliverAsync(candidates[index], _pump.ShutdownToken); + var (entry, message) = candidates[index]; + attempts[index] = DeliverAsync(entry, message, _pump.ShutdownToken); } var deliveries = Task.WhenAll(attempts); @@ -236,13 +363,14 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) foreach (var outcome in outcomes) { // Outcomes for entries which were replaced or removed since the attempt started are stale. - var id = outcome.Entry.Message.Envelope.MessageId; + var id = outcome.Entry.MessageId; if (!_messages.TryGetValue(id, out var current) || !ReferenceEquals(current, outcome.Entry)) continue; var status = outcome.Result?.Status; if (status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.DeadLettered) { Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id }); + Retire(current, status == DeliveryStatus.DeadLettered ? OutboxMessageRetirement.DeadLettered : OutboxMessageRetirement.Delivered); delivered++; } else @@ -250,6 +378,7 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) if (status == DeliveryStatus.Backpressured) backpressured++; var (retry, deadLetter) = PrepareRetry(outcome, ref deadLetters); Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id, State = retry, DeadLetter = deadLetter }); + if (deadLetter is not null) Retire(current, OutboxMessageRetirement.DeadLettered); } RecordDeliveryMetrics(outcome); applied++; @@ -265,6 +394,33 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) LogDeliveryComplete(Logger, delivered, backpressured, applied - delivered - backpressured, Count); } + /// Removes reference entries whose sources withdrew their messages, without delivering or retiring them. + private async ValueTask RemoveWithdrawnAsync(List withdrawn) + { + try + { + foreach (var entry in withdrawn) + { + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = entry.MessageId }); + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "withdrawn"); + if (entry.Source is null) + { + LogMessageUnprovided(Logger, entry.MessageId); + } + else + { + LogMessageWithdrawn(Logger, entry.MessageId); + } + } + } + catch (Exception exception) + { + FailStaging(exception); + } + + await PersistAsync().ConfigureAwait(true); + } + private void ApplyDeadLetters(Dictionary entries) { foreach (var key in _deadLetters.Keys.Where(key => !entries.ContainsKey(key)).ToArray()) @@ -299,7 +455,7 @@ private void ApplyDeadLetters(Dictionary entr { var letter = new OutboxDeadLetter { - Message = outcome.Entry.Message, + Message = outcome.Message, DeadLetteredAt = now, Reason = error, AttemptCount = attemptCount @@ -323,7 +479,7 @@ private void ApplyDeadLetters(Dictionary entr private void RecordDeliveryMetrics(DeliveryOutcome outcome) { - var message = outcome.Entry.Message; + var message = outcome.Message; var status = outcome.Result?.Status; var metricStatus = status switch { @@ -444,8 +600,14 @@ internal async Task StartAsync(CancellationToken cancellationToken) [LoggerMessage(Level = LogLevel.Error, Message = "Error delivering message {MessageId} to {ReceiverId}")] private static partial void LogDeliveryError(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + [LoggerMessage(Level = LogLevel.Information, Message = "Removed outbox message {MessageId}, which its source withdrew")] + private static partial void LogMessageWithdrawn(ILogger logger, HierarchicalKey messageId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Removed outbox message {MessageId}, which no registered outbox message source provides")] + private static partial void LogMessageUnprovided(ILogger logger, HierarchicalKey messageId); + [LoggerMessage(Level = LogLevel.Information, Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); - private readonly record struct DeliveryOutcome(OutboxEntry Entry, DeliveryResult? Result, string? Error, TimeSpan Duration, bool Expired); + private readonly record struct DeliveryOutcome(OutboxEntry Entry, OutboxMessage Message, DeliveryResult? Result, string? Error, TimeSpan Duration, bool Expired); } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 3ce8a87b33e..f18a62fb78c 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -110,13 +110,21 @@ construction or activation setup, before journal recovery. - `IOutbox.Register(IOutboxMessageSource)` adds a source, and `IOutbox.Send(messageId)` stages a reference entry with the message ID and delivery - state. The first source whose `TryGetMessage` returns the message owns it, and - returns the same `OutboxMessage` while it provides the message. A source withdraws - a pending message by returning `false`. When the message retires, `OnRetired` - reports `OutboxMessageRetirement.Delivered` or `DeadLettered`. After the send, a - source that throws or returns an invalid message dead-letters that entry alone, - with `OnRetired(DeadLettered)` in the same journal write; an exception during - `Send` propagates without staging anything. + state. The first source whose `TryGetMessage` returns the message owns it; the + outbox calls `TryGetMessage` to validate the send and before each delivery attempt, + and the source returns the same `OutboxMessage` while it provides the message. Every + delivery rule above applies. A source withdraws a pending message by returning + `false`: the outbox removes the entry before its next attempt, without delivery or + `OnRetired`, and an attempt already in flight still completes. When the message + retires, the outbox calls `OnRetired` with + `OutboxMessageRetirement.Delivered` (`Accepted` or `Duplicate`) or `DeadLettered` + (receiver-reported or outbox dead-lettering) while it stages the removal, so the + source's changes commit in the same journal write. An outbox dead letter stores the + materialized message. Sending a pending reference again coalesces; an ID pending as + an ordinary message, or one which no source provides, throws + `InvalidOperationException`. A source message addressed to the sending grain throws + `ArgumentException`. A recovered reference fails the activation when no source is + registered. - `IInbox.Register(IInboxMessageSink)` adds a sink. After validation and deduplication, the inbox offers each message, including self-sent and scheduled messages, to the sinks in order. The first sink whose `TryAccept` returns `true` @@ -193,6 +201,6 @@ delivered messages tagged by grain type and status; histograms for processing an delivery duration; and `orleans-messaging-inbox-depth` and `orleans-messaging-outbox-depth` up-down counters for pending work. A message a grain sends to itself counts as sent and as received by its inbox, and never contributes -to outbox delivery or depth. A message an inbox sink accepts counts as received -`accepted` and processed `success`; one which the sink dead-letters counts as received -and processed `dead_lettered`. +to outbox delivery or depth. Reference sends count as sent. A message an inbox sink +accepts counts as received `accepted` and processed `success`; one which the sink +dead-letters counts as received and processed `dead_lettered`. diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index a31cee8dd1d..72bd873656d 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -77,10 +77,31 @@ public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWh StringComparison.Ordinal); } + [Fact] + public void ReferenceEnqueue_CarriesNoMessageInJsonAndOrleansSerialization() + { + var command = new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() { EnqueuedAt = Now } }; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService>(); + + var json = JsonSerializer.Serialize(command, OutboxJsonContext.Default.OutboxCommand); + var fromJson = JsonSerializer.Deserialize(json, OutboxJsonContext.Default.OutboxCommand)!; + var fromBinary = serializer.Deserialize(serializer.SerializeToArray(command))!; + + Assert.DoesNotContain("Payload", json, StringComparison.Ordinal); + foreach (var recovered in new[] { fromJson, fromBinary }) + { + Assert.Null(recovered.Message); + Assert.Equal(HierarchicalKey.Create("orders", "reference"), recovered.Reference!.Value.MessageId); + } + } + private static InboxMessage InboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt) }; private static OutboxMessage OutboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt), ReceiverId = Receiver }; + private static OutboxMessageReference Reference(string id) => new() { MessageId = HierarchicalKey.Create("orders", id) }; + private static Envelope Envelope(string id, DateTimeOffset? deliverAt = null) => new() { MessageId = MessageId.Create("orders", id), @@ -126,6 +147,7 @@ private static OutboxCommand[] CreateOutboxCommands() [ new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() { EnqueuedAt = Now } }, new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() { EnqueuedAt = Now } }, + new() { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() { EnqueuedAt = Now } }, new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", EnqueuedAt = Now } }, new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "dead"), DeadLetter = deadLetter }, new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [HierarchicalKey.Create("orders", "dead")] }, @@ -134,7 +156,11 @@ private static OutboxCommand[] CreateOutboxCommands() Kind = OutboxCommandKind.Snapshot, Snapshot = new() { - Messages = [new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, EnqueuedAt = Now } }], + Messages = + [ + new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, EnqueuedAt = Now } }, + new() { Reference = Reference("pending-reference"), State = new() { AttemptCount = 2, EnqueuedAt = Now } } + ], DeadLetters = [deadLetter] } } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 60542564091..01d7919c20e 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -27,7 +27,222 @@ public abstract class MessageSourceAndSinkTests(MessageSourceAndSinkFixture fixt private SourceSinkProbe Probe => fixture.Probe; [Fact] - public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndTheSinkReportsLaterDuplicates() + public async Task SourceMessage_JournalsEachPayloadOnceAndRetiresInTheRemovalWrite() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("delivered"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Writes(sender).Length == 2); + var senderState = await OnGrainAsync(sender, context => Grain(context).GetState()); + Assert.Empty(senderState.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirement.Delivered), Assert.Single(senderState.Retired).Value); + // The outbox materializes the message once to validate the send and once for the delivery attempt. + Assert.Equal(2, Probe.For(sender).Materializations); + + // The payload is journaled once by each grain: in the sender's source state and in the receiver's sink state. + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(sender), payload)); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + // The retirement commits in the write which removes the outbox entry. + var writes = Writes(sender); + var retirement = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirement.Delivered)); + Assert.True(writes[0].AsSpan().IndexOf(retirement) < 0); + Assert.True(writes[1].AsSpan().IndexOf(retirement) >= 0); + + // The sink accepted the message without an inbox entry; the inbox journals only its processed record. + var receiverState = await OnGrainAsync(receiver, context => ( + Grain(context).GetState(), + Inbox(context).Count, + InboxTestAccess.GetProcessedKeys(Inbox(context)).Contains(messageId))); + Assert.Equal(payload, receiverState.Item1.Received[messageId.ToString()]); + Assert.Equal(0, receiverState.Count); + Assert.True(receiverState.Item3); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(0, Probe.For(receiver).Handled); + Assert.Single(Writes(receiver)); + + // A recovered sender agrees: the entry is removed and its retirement is recorded. + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var senderContext)); + await sender.DeactivateAsync(); + await senderContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + var recovered = await OnGrainAsync(sender, context => Grain(context).GetState()); + Assert.Empty(recovered.Outgoing); + Assert.Single(recovered.Retired); + Assert.Equal(2, Writes(sender).Length); + } + + [Fact] + public async Task SourceMessage_RecoveredFromTheJournal_IsDeliveredFromItsSource() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("recovered"); + var payload = NewPayload(); + + await sender.StageFromSourceAndDeactivateAsync(receiver.GetGrainId(), messageId, payload); + + await WaitUntilAsync(receiver, context => Grain(context).GetState().Received.ContainsKey(messageId.ToString())); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(sender), payload)); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + } + + [Fact] + public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBeforeRetiring() + { + var sender = NewGrain(); + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + var messageId = NewMessageId("outbox-dead-letter"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + // The receiver has no handler, so both permitted delivery attempts fail. + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, deadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[messageId.ToString()]); + var letter = Assert.Single(deadLetters); + Assert.Equal(messageId, letter.Message.Envelope.MessageId); + Assert.Equal(receiver.GetGrainId(), letter.Message.ReceiverId); + Assert.Equal(payload, letter.Message.Envelope.Payload); + Assert.Equal(SourceSinkGrain.Subject, letter.Message.Envelope.Subject); + Assert.Equal(2, letter.AttemptCount); + Assert.Equal(3, Probe.For(sender).Materializations); + + // The dead letter keeps its own copy for diagnosis. + await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 2); + } + + [Fact] + public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsDeadLettered() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.Throw; + var messageId = NewMessageId("receiver-dead-letter"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, outboxDeadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[messageId.ToString()]); + Assert.Empty(state.Outgoing); + Assert.Empty(outboxDeadLetters); + var inboxLetter = Assert.Single(await OnGrainAsync(receiver, context => + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Equal(payload, inboxLetter.Message.Envelope.Payload); + Assert.Equal(SourceSinkGrain.Rejection, inboxLetter.Reason); + } + + [Fact] + public async Task SourceMessage_CoalescesWithItsPendingReferenceAndRejectsConflicts() + { + var sender = NewGrain(); + var receiver = NewGrain().GetGrainId(); + var messageId = NewMessageId("coalesced"); + var sentId = NewMessageId("sent"); + + await sender.StageFromSourceAsync(receiver, messageId, NewPayload(), write: false); + await sender.SendFromSourceAsync(messageId); + Assert.Equal(1, await sender.GetOutboxCountAsync()); + + await Assert.ThrowsAsync(() => sender.SendAsync(receiver, messageId, NewPayload(), write: false)); + await sender.SendAsync(receiver, sentId, NewPayload(), write: false); + await Assert.ThrowsAsync(() => sender.SendFromSourceAsync(sentId)); + await Assert.ThrowsAsync(() => sender.SendFromSourceAsync(NewMessageId("unprovided"))); + await Assert.ThrowsAsync(() => sender.StageFromSourceAsync(sender.GetGrainId(), NewMessageId("self"), NewPayload(), write: false)); + Assert.Equal(2, await sender.GetOutboxCountAsync()); + + await sender.WriteAsync(); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0); + + // Materialized to validate the first send, to reject the self-addressed send, and for delivery; an unprovided + // message is never materialized. + Assert.Equal(3, Probe.For(sender).Materializations); + } + + [Fact] + public async Task SourceMessage_WithdrawnBySource_IsRemovedWithoutDeliveryOrRetirement() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("withdrawn"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: false); + await sender.WithdrawAsync(messageId); + Assert.Equal(1, await sender.GetOutboxCountAsync()); + await sender.WriteAsync(); + + // The removal commits in its own write, after the write which staged the entry. + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Writes(sender).Length == 2); + var state = await OnGrainAsync(sender, context => Grain(context).GetState()); + Assert.Empty(state.Outgoing); + Assert.Empty(state.Retired); + Assert.Equal(1, Probe.For(sender).Materializations); + Assert.Equal(0, Probe.For(receiver).SinkOffers); + + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var senderContext)); + await sender.DeactivateAsync(); + await senderContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + } + + [Fact] + public async Task RecoveredReferenceEntry_IsWithdrawnWhenNoSourceProvidesItAndFailsWithoutSources() + { + var grain = NewGrain(); + var withoutSources = Cluster.Client.GetGrain(Guid.NewGuid()); + _ = await grain.GetOutboxCountAsync(); + await withoutSources.PingAsync(); + var unprovided = NewMessageId("unprovided"); + + // Replays a reference entry, validates the recovered state as recovery does, then runs a delivery pass. + await OnGrainAsync(grain, context => + { + var outbox = Outbox(context); + ApplyReference(outbox, unprovided); + ((IStateMachine)outbox).OnRecoveryCompleted(); + var pump = context.ActivationServices.GetRequiredService(); + pump.EnsureStarted(); + pump.Signal(); + return true; + }); + await WaitUntilAsync(grain, context => Outbox(context).Count == 0); + Assert.Empty((await OnGrainAsync(grain, context => Grain(context).GetState())).Retired); + + var error = await Assert.ThrowsAsync(() => OnGrainAsync(withoutSources, context => + { + var outbox = Outbox(context); + ApplyReference(outbox, unprovided); + ((IStateMachine)outbox).OnRecoveryCompleted(); + return true; + })); + Assert.Contains("no outbox message source is registered", error.Message, StringComparison.Ordinal); + + void ApplyReference(Outbox outbox, HierarchicalKey messageId) => + ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand + { + Kind = OutboxCommandKind.Enqueue, + Reference = new() { MessageId = messageId }, + State = new() { EnqueuedAt = DateTimeOffset.UtcNow } + }); + } + + [Fact] + public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndIsOfferedAgainAfterItExpires() { var receiver = NewGrain(); var message = NewInboxMessage("deduplicated", NewPayload()); @@ -166,6 +381,26 @@ public async Task DeclinedMessages_UseInboxAcceptanceAndHandlers() Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), declinedPayload)); } + [Fact] + public async Task SelfSentMessage_IsAcceptedByTheSinkInTheSendingWrite() + { + var grain = NewGrain(); + _ = await grain.GetOutboxCountAsync(); + var writes = Writes(grain).Length; + var messageId = NewMessageId("self"); + var payload = NewPayload(); + + await grain.SendAsync(grain.GetGrainId(), messageId, payload, write: true); + + var (state, inboxCount, outboxCount) = await OnGrainAsync(grain, context => (Grain(context).GetState(), Inbox(context).Count, Outbox(context).Count)); + Assert.Equal(payload, state.Received[messageId.ToString()]); + Assert.Equal(0, inboxCount); + Assert.Equal(0, outboxCount); + Assert.Equal(1, Probe.For(grain).SinkAcceptances); + Assert.Equal(writes + 1, Writes(grain).Length); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(grain), payload)); + } + private ISourceSinkGrain NewGrain() => Cluster.Client.GetGrain(Guid.NewGuid()); private static MessageId NewMessageId(string name) => MessageId.Create("source-sink", name, Guid.NewGuid().ToString("N")); @@ -185,6 +420,8 @@ public async Task DeclinedMessages_UseInboxAcceptanceAndHandlers() private static Inbox Inbox(IGrainContext context) => context.ActivationServices.GetRequiredService(); + private static Outbox Outbox(IGrainContext context) => context.ActivationServices.GetRequiredService(); + private Task OnGrainAsync(IAddressable grain, Func func) { if (!Cluster.TryGetGrainContext(grain.GetGrainId(), out var context)) @@ -340,17 +577,25 @@ public sealed class Counters public interface ISourceSinkGrain : IGrainWithGuidKey { + Task StageFromSourceAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write); + Task StageFromSourceAndDeactivateAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload); + Task SendFromSourceAsync(HierarchicalKey messageId); + Task WithdrawAsync(HierarchicalKey messageId); + Task SendAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write); Task WriteAsync(); + Task GetOutboxCountAsync(); Task DeactivateAsync(); } -/// Stores received payloads as an inbox sink. -public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, IInboxMessageSink +/// Stores outgoing messages as an outbox source and received payloads as an inbox sink. +public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, IOutboxMessageSource, IInboxMessageSink { internal const string Subject = "source-sink/payload"; internal const string Rejection = "The sink rejected the message."; private readonly IJournaledStateManager _manager; + private readonly IOutbox _outbox; private readonly IDurableDictionary _outgoing; + private readonly IDurableDictionary _receivers; private readonly IDurableDictionary _received; private readonly IDurableDictionary _retired; private readonly SourceSinkProbe.Counters _probe; @@ -359,32 +604,101 @@ public SourceSinkGrain( IGrainContext context, IJournaledStateManager manager, IInbox inbox, + IOutbox outbox, [FromKeyedServices("outgoing")] IDurableDictionary outgoing, + [FromKeyedServices("receivers")] IDurableDictionary receivers, [FromKeyedServices("received")] IDurableDictionary received, [FromKeyedServices("retired")] IDurableDictionary retired, SourceSinkProbe probe) { _manager = manager; + _outbox = outbox; _outgoing = outgoing; + _receivers = receivers; _received = received; _retired = retired; _probe = probe.For(context.GrainId); - _probe.OnActivated(); + outbox.Register(this); inbox.Register(this).Register(new Handler(_probe)); } internal (Dictionary Outgoing, Dictionary Received, Dictionary Retired) GetState() => (new(_outgoing), new(_received), new(_retired)); + public async Task StageFromSourceAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write) + { + _outgoing[messageId.ToString()] = payload; + _receivers[messageId.ToString()] = receiver.ToString(); + _outbox.Send(messageId); + if (write) await _manager.WriteStateAsync(CancellationToken.None); + } + + public async Task StageFromSourceAndDeactivateAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload) + { + await StageFromSourceAsync(receiver, messageId, payload, write: true); + DeactivateOnIdle(); + } + + public Task SendFromSourceAsync(HierarchicalKey messageId) + { + _outbox.Send(messageId); + return Task.CompletedTask; + } + + public Task WithdrawAsync(HierarchicalKey messageId) + { + Assert.True(_outgoing.Remove(messageId.ToString())); + Assert.True(_receivers.Remove(messageId.ToString())); + return Task.CompletedTask; + } + + public async Task SendAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write) + { + _outbox.Send(new OutboxMessage + { + Envelope = new Envelope { MessageId = messageId, Payload = payload, Subject = Subject }, + ReceiverId = receiver + }); + if (write) await _manager.WriteStateAsync(CancellationToken.None); + } + public async Task WriteAsync() => await _manager.WriteStateAsync(CancellationToken.None); + public Task GetOutboxCountAsync() => Task.FromResult(_outbox.Count()); + public Task DeactivateAsync() { DeactivateOnIdle(); return Task.CompletedTask; } - bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result) + bool IOutboxMessageSource.TryGetMessage(HierarchicalKey messageId, out OutboxMessage message) + { + var key = messageId.ToString(); + if (!_outgoing.TryGetValue(key, out var payload)) + { + message = default; + return false; + } + + _probe.OnMaterialized(); + message = new() + { + Envelope = new() { MessageId = messageId, Payload = payload, Subject = Subject }, + ReceiverId = GrainId.Parse(_receivers[key]) + }; + return true; + } + + void IOutboxMessageSource.OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement) + { + var key = messageId.ToString(); + Assert.True(_outgoing.Remove(key)); + Assert.True(_receivers.Remove(key)); + _retired[key] = retirement.ToString(); + } + + bool IInboxMessageSink.TryAccept(InboxMessage message) { result = default; if (message.Envelope.Subject != Subject) return false; diff --git a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs index f2870bcca47..3dfdf3df897 100644 --- a/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs +++ b/test/Orleans.Messaging.Tests/Support/InboxTestAccess.cs @@ -16,7 +16,7 @@ internal static class InboxTestAccess public static IEnumerable Messages(this IOutbox outbox) => outbox switch { - Outbox value => value.MessageState.Values.Select(static entry => entry.Message), + Outbox value => value.MessageState.Values.Where(static entry => entry.Message is not null).Select(static entry => entry.Message!.Value), JournaledTestOutbox value => value.Messages, _ => throw new NotSupportedException(outbox.GetType().FullName) }; From 0281364cd588d786be81ed391448fc1f2e679e45 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 12:45:03 -0700 Subject: [PATCH 71/95] feat(messaging): identify messages with MessageId wrapping HierarchicalKey Outbox messages, reference entries, dead letters, and their diagnostics identify messages with MessageId. --- src/Orleans.Messaging/Outbox.State.cs | 10 +++--- src/Orleans.Messaging/Outbox.cs | 34 +++++++++---------- .../MessagingCommandSerializationTests.cs | 10 +++--- .../Functional/JsonJournalMessagingTests.cs | 18 +++++----- .../Functional/MessageSourceAndSinkTests.cs | 26 +++++++------- .../MessagingGrainTypeConfiguratorTests.cs | 2 +- .../MessagingProviderCutoverTests.cs | 2 +- .../MultiSiloMessagingFailoverTests.cs | 2 +- .../Functional/OutboxDeliveryBehaviorTests.cs | 4 +-- .../PublicInboxHandlerTransactionTests.cs | 14 ++++---- .../Functional/SelfAddressedMessageTests.cs | 2 +- .../Support/BootstrapOutboxServices.cs | 24 ++++++------- .../Support/BootstrapTestGrains.cs | 8 ++--- .../Support/MessagingTestGrains.cs | 2 +- .../Support/OutboxStateTestAccess.cs | 8 ++--- .../Support/OutboxTestMessages.cs | 4 +-- .../Support/RawPayloadTestGrain.cs | 8 ++--- 17 files changed, 89 insertions(+), 89 deletions(-) diff --git a/src/Orleans.Messaging/Outbox.State.cs b/src/Orleans.Messaging/Outbox.State.cs index 4647933c4b3..41fee1c43d5 100644 --- a/src/Orleans.Messaging/Outbox.State.cs +++ b/src/Orleans.Messaging/Outbox.State.cs @@ -98,7 +98,7 @@ private void Apply(OutboxCommand command) } } - internal bool RemoveDeadLetter(HierarchicalKey id) + internal bool RemoveDeadLetter(MessageId id) { _pump.ShutdownToken.ThrowIfCancellationRequested(); if (!_deadLetters.ContainsKey(id)) return false; @@ -120,10 +120,10 @@ internal sealed class OutboxCommand { [Id(0)] public OutboxCommandKind Kind { get; init; } [Id(1)] public OutboxMessage? Message { get; init; } - [Id(2)] public HierarchicalKey MessageId { get; init; } + [Id(2)] public MessageId MessageId { get; init; } [Id(3)] public OutboxMessageState? State { get; init; } [Id(4)] public OutboxDeadLetter? DeadLetter { get; init; } - [Id(5)] public HierarchicalKey[]? RemovedDeadLetters { get; init; } + [Id(5)] public MessageId[]? RemovedDeadLetters { get; init; } [Id(6)] public OutboxSnapshot? Snapshot { get; init; } // An enqueue command sets this instead of Message for a message which an IOutboxMessageSource provides. @@ -136,7 +136,7 @@ internal sealed class OutboxCommand [GenerateSerializer, Alias("Orleans.Messaging.OutboxMessageReference")] internal readonly struct OutboxMessageReference { - [Id(0)] public required HierarchicalKey MessageId { get; init; } + [Id(0)] public required MessageId MessageId { get; init; } } [GenerateSerializer, Alias("Orleans.Messaging.OutboxSnapshot")] @@ -168,7 +168,7 @@ internal sealed class OutboxEntry [JsonIgnore] public IOutboxMessageSource? Source { get; set; } [JsonIgnore] - public HierarchicalKey MessageId => Message is { } message ? message.Envelope.MessageId + public MessageId MessageId => Message is { } message ? message.Envelope.MessageId : Reference is { } reference ? reference.MessageId : throw new InvalidOperationException("An outbox entry requires its message or message reference."); } diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 65e522ab6f8..f3a7dde8794 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -20,8 +20,8 @@ internal sealed partial class Outbox : IStateMachine, IDurableValueCommandHandle { private readonly MessagingPump _pump; private JournalStreamWriter _writer; - private readonly Dictionary _messages = []; - private readonly Dictionary _deadLetters = []; + private readonly Dictionary _messages = []; + private readonly Dictionary _deadLetters = []; private readonly List _sources = []; private Task? _deliveries; @@ -39,8 +39,8 @@ public Outbox(MessagingPump pump) private TimeProvider TimeProvider => _pump.Shared.JobTimeProvider; private GrainId GrainId => _pump.GrainContext.GrainId; - internal IReadOnlyDictionary MessageState => _messages; - internal IReadOnlyDictionary DeadLetterState => _deadLetters; + internal IReadOnlyDictionary MessageState => _messages; + internal IReadOnlyDictionary DeadLetterState => _deadLetters; internal int Count => _messages.Count; public IOutbox Register(IOutboxMessageSource source) @@ -83,7 +83,7 @@ public void Send(OutboxMessage message) } } - public void Send(HierarchicalKey messageId) + public void Send(MessageId messageId) { _pump.ShutdownToken.ThrowIfCancellationRequested(); EnvelopeValidation.ValidateMessageId(messageId); @@ -121,7 +121,7 @@ public void Send(HierarchicalKey messageId) } /// Returns the first registered source which provides the message, and its validated message. - private IOutboxMessageSource? FindSource(HierarchicalKey messageId, out OutboxMessage message) + private IOutboxMessageSource? FindSource(MessageId messageId, out OutboxMessage message) { foreach (var source in _sources) { @@ -149,7 +149,7 @@ private bool TryGetMessage(OutboxEntry entry, out OutboxMessage message) return entry.Source is not null; } - private bool TryGetSourceMessage(IOutboxMessageSource source, HierarchicalKey messageId, out OutboxMessage message) + private bool TryGetSourceMessage(IOutboxMessageSource source, MessageId messageId, out OutboxMessage message) { if (!source.TryGetMessage(messageId, out message)) return false; EnvelopeValidation.Validate(message); @@ -356,7 +356,7 @@ private async Task SignalOnCompletionAsync(Task deliveries) private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) { - Dictionary? deadLetters = null; + Dictionary? deadLetters = null; int applied = 0, delivered = 0, backpressured = 0; try { @@ -421,7 +421,7 @@ private async ValueTask RemoveWithdrawnAsync(List withdrawn) await PersistAsync().ConfigureAwait(true); } - private void ApplyDeadLetters(Dictionary entries) + private void ApplyDeadLetters(Dictionary entries) { foreach (var key in _deadLetters.Keys.Where(key => !entries.ContainsKey(key)).ToArray()) { @@ -438,7 +438,7 @@ private void ApplyDeadLetters(Dictionary entr private (OutboxMessageState? Retry, OutboxDeadLetter? DeadLetter) PrepareRetry( DeliveryOutcome outcome, - ref Dictionary? deadLetters) + ref Dictionary? deadLetters) { var previous = outcome.Entry.State; var status = outcome.Result?.Status; @@ -586,25 +586,25 @@ internal async Task StartAsync(CancellationToken cancellationToken) private static partial void LogDeliveringMessages(ILogger logger, int count); [LoggerMessage(Level = LogLevel.Debug, Message = "Delivered message {MessageId} to {ReceiverId} (Status: {Status})")] - private static partial void LogMessageDelivered(ILogger logger, HierarchicalKey messageId, GrainId receiverId, DeliveryStatus status); + private static partial void LogMessageDelivered(ILogger logger, MessageId messageId, GrainId receiverId, DeliveryStatus status); [LoggerMessage(Level = LogLevel.Warning, Message = "No inbox handler for message {MessageId} to {ReceiverId}: {Message}")] - private static partial void LogDeliveryHandlerNotFound(ILogger logger, HierarchicalKey messageId, GrainId receiverId, string? message); + private static partial void LogDeliveryHandlerNotFound(ILogger logger, MessageId messageId, GrainId receiverId, string? message); [LoggerMessage(Level = LogLevel.Debug, Message = "Backpressured delivering message {MessageId} to {ReceiverId}, will retry later")] - private static partial void LogDeliveryBackpressured(ILogger logger, HierarchicalKey messageId, GrainId receiverId); + private static partial void LogDeliveryBackpressured(ILogger logger, MessageId messageId, GrainId receiverId); [LoggerMessage(Level = LogLevel.Warning, Message = "Unexpected delivery status {Status} for message {MessageId} to {ReceiverId}")] - private static partial void LogUnexpectedDeliveryStatus(ILogger logger, DeliveryStatus status, HierarchicalKey messageId, GrainId receiverId); + private static partial void LogUnexpectedDeliveryStatus(ILogger logger, DeliveryStatus status, MessageId messageId, GrainId receiverId); [LoggerMessage(Level = LogLevel.Error, Message = "Error delivering message {MessageId} to {ReceiverId}")] - private static partial void LogDeliveryError(ILogger logger, Exception exception, HierarchicalKey messageId, GrainId receiverId); + private static partial void LogDeliveryError(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); [LoggerMessage(Level = LogLevel.Information, Message = "Removed outbox message {MessageId}, which its source withdrew")] - private static partial void LogMessageWithdrawn(ILogger logger, HierarchicalKey messageId); + private static partial void LogMessageWithdrawn(ILogger logger, MessageId messageId); [LoggerMessage(Level = LogLevel.Warning, Message = "Removed outbox message {MessageId}, which no registered outbox message source provides")] - private static partial void LogMessageUnprovided(ILogger logger, HierarchicalKey messageId); + private static partial void LogMessageUnprovided(ILogger logger, MessageId messageId); [LoggerMessage(Level = LogLevel.Information, Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 72bd873656d..646951c4b8a 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -92,7 +92,7 @@ public void ReferenceEnqueue_CarriesNoMessageInJsonAndOrleansSerialization() foreach (var recovered in new[] { fromJson, fromBinary }) { Assert.Null(recovered.Message); - Assert.Equal(HierarchicalKey.Create("orders", "reference"), recovered.Reference!.Value.MessageId); + Assert.Equal(MessageId.Create("orders", "reference"), recovered.Reference!.Value.MessageId); } } @@ -100,7 +100,7 @@ public void ReferenceEnqueue_CarriesNoMessageInJsonAndOrleansSerialization() private static OutboxMessage OutboxMessage(string id, DateTimeOffset? deliverAt = null) => new() { Envelope = Envelope(id, deliverAt), ReceiverId = Receiver }; - private static OutboxMessageReference Reference(string id) => new() { MessageId = HierarchicalKey.Create("orders", id) }; + private static OutboxMessageReference Reference(string id) => new() { MessageId = MessageId.Create("orders", id) }; private static Envelope Envelope(string id, DateTimeOffset? deliverAt = null) => new() { @@ -148,9 +148,9 @@ private static OutboxCommand[] CreateOutboxCommands() new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() { EnqueuedAt = Now } }, new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() { EnqueuedAt = Now } }, new() { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() { EnqueuedAt = Now } }, - new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", EnqueuedAt = Now } }, - new() { Kind = OutboxCommandKind.Delivery, MessageId = HierarchicalKey.Create("orders", "dead"), DeadLetter = deadLetter }, - new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [HierarchicalKey.Create("orders", "dead")] }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", EnqueuedAt = Now } }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "dead"), DeadLetter = deadLetter }, + new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [MessageId.Create("orders", "dead")] }, new() { Kind = OutboxCommandKind.Snapshot, diff --git a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs index 494e42567d6..f33bfbd1b9d 100644 --- a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs @@ -24,7 +24,7 @@ public async Task JsonJournal_RecoversOutboxAndInboxStateAcrossActivations() { var sender = Cluster.Client.GetGrain(Guid.NewGuid()); var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); - var messageId = HierarchicalKey.Create("json", Guid.NewGuid().ToString("N")); + var messageId = MessageId.Create("json", Guid.NewGuid().ToString("N")); // The sender deactivates after committing the send, so delivery runs from the outbox recovered from the JSON journal. await sender.SendAndDeactivateAsync(receiver.GetGrainId(), messageId, 42); @@ -52,7 +52,7 @@ public async Task CustomInboxHandlers_FirstHandlerWhichCanHandleReceivesMessage( var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); var message = new InboxMessage { - Envelope = new Envelope { MessageId = HierarchicalKey.Create("custom", Guid.NewGuid().ToString("N")), Payload = [7], Subject = "custom" } + Envelope = new Envelope { MessageId = MessageId.Create("custom", Guid.NewGuid().ToString("N")), Payload = [7], Subject = "custom" } }; var result = await receiver.AsReference().DeliverAsync(message, Token); @@ -65,7 +65,7 @@ public async Task CustomInboxHandlers_FirstHandlerWhichCanHandleReceivesMessage( public async Task JsonJournal_RecoversScheduledInboxMessageWithItsDeliverAt() { var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); - var messageId = HierarchicalKey.Create("json", "scheduled", Guid.NewGuid().ToString("N")); + var messageId = MessageId.Create("json", "scheduled", Guid.NewGuid().ToString("N")); var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); var deliverAt = new DateTimeOffset(DateTime.UtcNow.AddDays(1).Ticks, TimeSpan.Zero).ToOffset(TimeSpan.FromHours(5)); var scheduled = type.Create(messageId, receiver.GetGrainId(), 7, deliverAt).ToInboxMessage(); @@ -93,8 +93,8 @@ public async Task JsonJournal_RecoversScheduledInboxMessageWithItsDeliverAt() public async Task JsonJournal_RecoversSelfSentMessagesFromTheInbox() { var grain = Cluster.Client.GetGrain(Guid.NewGuid()); - var scheduledId = HierarchicalKey.Create("json", "self-scheduled", Guid.NewGuid().ToString("N")); - var immediateId = HierarchicalKey.Create("json", "self", Guid.NewGuid().ToString("N")); + var scheduledId = MessageId.Create("json", "self-scheduled", Guid.NewGuid().ToString("N")); + var immediateId = MessageId.Create("json", "self", Guid.NewGuid().ToString("N")); var deliverAt = new DateTimeOffset(DateTime.UtcNow.AddDays(1).Ticks, TimeSpan.Zero); // Each send commits the acceptance in the grain's own inbox; the outbox never holds the message. @@ -171,8 +171,8 @@ public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken can } public interface IJsonMessagingGrain : IGrainWithGuidKey { - Task SendAsync(GrainId destination, HierarchicalKey messageId, int value, DateTimeOffset? deliverAt = null); - Task SendAndDeactivateAsync(GrainId destination, HierarchicalKey messageId, int value); + Task SendAsync(GrainId destination, MessageId messageId, int value, DateTimeOffset? deliverAt = null); + Task SendAndDeactivateAsync(GrainId destination, MessageId messageId, int value); Task GetOutboxCountAsync(); Task DeactivateAsync(); } @@ -219,13 +219,13 @@ public JsonMessagingGrain(IJournaledStateManager manager, IInbox inbox, IOutbox }); } - public async Task SendAsync(GrainId destination, HierarchicalKey messageId, int value, DateTimeOffset? deliverAt = null) + public async Task SendAsync(GrainId destination, MessageId messageId, int value, DateTimeOffset? deliverAt = null) { _outbox.Send(_type, messageId, destination, value, deliverAt); await _manager.WriteStateAsync(CancellationToken.None); } - public async Task SendAndDeactivateAsync(GrainId destination, HierarchicalKey messageId, int value) + public async Task SendAndDeactivateAsync(GrainId destination, MessageId messageId, int value) { await SendAsync(destination, messageId, value); DeactivateOnIdle(); diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 01d7919c20e..bdf42c7103c 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -232,7 +232,7 @@ await OnGrainAsync(grain, context => })); Assert.Contains("no outbox message source is registered", error.Message, StringComparison.Ordinal); - void ApplyReference(Outbox outbox, HierarchicalKey messageId) => + void ApplyReference(Outbox outbox, MessageId messageId) => ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand { Kind = OutboxCommandKind.Enqueue, @@ -577,11 +577,11 @@ public sealed class Counters public interface ISourceSinkGrain : IGrainWithGuidKey { - Task StageFromSourceAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write); - Task StageFromSourceAndDeactivateAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload); - Task SendFromSourceAsync(HierarchicalKey messageId); - Task WithdrawAsync(HierarchicalKey messageId); - Task SendAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write); + Task StageFromSourceAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write); + Task StageFromSourceAndDeactivateAsync(GrainId receiver, MessageId messageId, byte[] payload); + Task SendFromSourceAsync(MessageId messageId); + Task WithdrawAsync(MessageId messageId); + Task SendAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write); Task WriteAsync(); Task GetOutboxCountAsync(); Task DeactivateAsync(); @@ -625,7 +625,7 @@ public SourceSinkGrain( internal (Dictionary Outgoing, Dictionary Received, Dictionary Retired) GetState() => (new(_outgoing), new(_received), new(_retired)); - public async Task StageFromSourceAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write) + public async Task StageFromSourceAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write) { _outgoing[messageId.ToString()] = payload; _receivers[messageId.ToString()] = receiver.ToString(); @@ -633,26 +633,26 @@ public async Task StageFromSourceAsync(GrainId receiver, HierarchicalKey message if (write) await _manager.WriteStateAsync(CancellationToken.None); } - public async Task StageFromSourceAndDeactivateAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload) + public async Task StageFromSourceAndDeactivateAsync(GrainId receiver, MessageId messageId, byte[] payload) { await StageFromSourceAsync(receiver, messageId, payload, write: true); DeactivateOnIdle(); } - public Task SendFromSourceAsync(HierarchicalKey messageId) + public Task SendFromSourceAsync(MessageId messageId) { _outbox.Send(messageId); return Task.CompletedTask; } - public Task WithdrawAsync(HierarchicalKey messageId) + public Task WithdrawAsync(MessageId messageId) { Assert.True(_outgoing.Remove(messageId.ToString())); Assert.True(_receivers.Remove(messageId.ToString())); return Task.CompletedTask; } - public async Task SendAsync(GrainId receiver, HierarchicalKey messageId, byte[] payload, bool write) + public async Task SendAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write) { _outbox.Send(new OutboxMessage { @@ -672,7 +672,7 @@ public Task DeactivateAsync() return Task.CompletedTask; } - bool IOutboxMessageSource.TryGetMessage(HierarchicalKey messageId, out OutboxMessage message) + bool IOutboxMessageSource.TryGetMessage(MessageId messageId, out OutboxMessage message) { var key = messageId.ToString(); if (!_outgoing.TryGetValue(key, out var payload)) @@ -690,7 +690,7 @@ bool IOutboxMessageSource.TryGetMessage(HierarchicalKey messageId, out OutboxMes return true; } - void IOutboxMessageSource.OnRetired(HierarchicalKey messageId, OutboxMessageRetirement retirement) + void IOutboxMessageSource.OnRetired(MessageId messageId, OutboxMessageRetirement retirement) { var key = messageId.ToString(); Assert.True(_outgoing.Remove(key)); diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs index 5a42533a25f..b9853aadf48 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs @@ -460,7 +460,7 @@ private IBootstrapTestGrain CreateGrain(Type grainClass) => grainClass == typeof private InboxMessage CreateEnvelope(IAddressable grain) => OutboxTestMessages.CreateApplication(Fixture.Client.ServiceProvider, - HierarchicalKey.Create("test", grain.GetGrainId().ToString(), "command", (++_commandSequence).ToString(CultureInfo.InvariantCulture)), + MessageId.Create("test", grain.GetGrainId().ToString(), "command", (++_commandSequence).ToString(CultureInfo.InvariantCulture)), GrainId.Create("bootstrap-sender", "external"), grain.GetGrainId(), BootstrapState.Route, 1).ToInboxMessage(); private static int GetProcessedCount(BootstrapObservation observation) => diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs index ec8570a36ee..53d58160bcf 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingProviderCutoverTests.cs @@ -60,7 +60,7 @@ public async Task PublicHosting_NamedFactories_CommitsCompositeEndpointsAndRecov var sender = cluster.Client.GetGrain(Guid.NewGuid()); var receiver = cluster.Client.GetGrain(Guid.NewGuid()); var sink = cluster.Client.GetGrain(Guid.NewGuid()); - var logicalId = HierarchicalKey.Create("cutover", Guid.NewGuid().ToString("N")); + var logicalId = MessageId.Create("cutover", Guid.NewGuid().ToString("N")); var receivedTask = snapshots.WaitAsync(receiver.GetGrainId(), Token); var forwardedTask = snapshots.WaitAsync(sink.GetGrainId(), Token); await sender.SendAsync(receiver.GetGrainId(), "messages/record", new TestMessage(logicalId, 1, "composite", sink.GetGrainId())); diff --git a/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs index b4ff786accb..a1e8bb36b81 100644 --- a/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MultiSiloMessagingFailoverTests.cs @@ -21,7 +21,7 @@ public async Task ReceiverSiloStops_DuringBlockedHandler_NewActivationRecoversFr var receiver = fixture.Client.GetGrain(Guid.NewGuid()); using var barrier = fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/failover"); var sender = fixture.Client.GetGrain(Guid.NewGuid()); - var logicalId = HierarchicalKey.Create("failover", Guid.NewGuid().ToString("N")); + var logicalId = MessageId.Create("failover", Guid.NewGuid().ToString("N")); fixture.JobManagerProbe.DuplicateNext(MessagingPump.JobName, receiver.GetGrainId()); await sender.SendAsync( diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs index 3cec0f71534..e93914066f5 100644 --- a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryBehaviorTests.cs @@ -162,7 +162,7 @@ public async Task FailedAtomicWrite_FencesOldObjectsAndFreshReplayOmitsEffectAnd var oldContext = Fixture.GetGrainContext(sender); var oldManager = oldContext.ActivationServices.GetRequiredService(); var oldGrain = Assert.IsType(oldContext.GrainInstance); - await sender.StageEffectAsync(new MessageEffect(HierarchicalKey.Create("outbox", "failed-write"), 1, 53, "failed-write")); + await sender.StageEffectAsync(new MessageEffect(MessageId.Create("outbox", "failed-write"), 1, 53, "failed-write")); Fixture.Storage.FailWrite(JournalId.FromGrainId(sender.GetGrainId())); var failure = await Assert.ThrowsAsync(() => sender.SendAsync( @@ -202,7 +202,7 @@ await sender.StageWithoutCommitAsync( Assert.Equal(0, fresh.OutboxCount); Assert.Empty((await receiver.GetSnapshotAsync()).Effects); - var effect = new MessageEffect(HierarchicalKey.Create("outbox", "fresh-owner"), 1, 76, "fresh-owner"); + var effect = new MessageEffect(MessageId.Create("outbox", "fresh-owner"), 1, 76, "fresh-owner"); await sender.StageEffectAsync(effect); var drained = Fixture.SnapshotProbe.WaitAsync( sender.GetGrainId(), diff --git a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs index c03629d154a..628435b84c9 100644 --- a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs @@ -34,7 +34,7 @@ public async Task ArbitraryRawPayload_AcceptanceReplayForwardAndBusinessCompleti var bytes = System.Text.Encoding.UTF8.GetBytes("raw application bytes\0\u03c0/forward").Concat(new byte[] { 0xff, 0x80, 0x01 }).ToArray(); var envelope = new Envelope { - MessageId = HierarchicalKey.Create("raw", Guid.NewGuid().ToString("N")), + MessageId = MessageId.Create("raw", Guid.NewGuid().ToString("N")), Payload = bytes, Headers = new Dictionary(StringComparer.Ordinal) { ["attachment"] = [0xff, 0x80, 0x01] } }; @@ -126,8 +126,8 @@ public async Task ArbitraryRawPayload_AcceptanceReplayForwardAndBusinessCompleti public async Task CaptureProbeSnapshot_RemainsStableAcrossLaterWrites() { var receiver = NewGrain(); - var first = new MessageEffect(HierarchicalKey.Create("capture", "first"), 1, 1, "first"); - var second = new MessageEffect(HierarchicalKey.Create("capture", "second"), 1, 2, "second"); + var first = new MessageEffect(MessageId.Create("capture", "first"), 1, 1, "first"); + var second = new MessageEffect(MessageId.Create("capture", "second"), 1, 2, "second"); await receiver.StageEffectAsync(first); await receiver.RetryWriteStateAsync(); var grain = Assert.IsType(Fixture.GetGrainContext(receiver).GrainInstance); @@ -157,7 +157,7 @@ public async Task HandlerSuccess_CommitsEffectCompletionDedupeAndOutgoingAtomica { var receiver = NewGrain(); var sink = NewGrain(); - var logicalId = HierarchicalKey.Create("atomic", Guid.NewGuid().ToString("N")); + var logicalId = MessageId.Create("atomic", Guid.NewGuid().ToString("N")); var envelope = CreateEnvelope( receiver, new TestMessage(logicalId, 7, "atomic", sink.GetGrainId())); @@ -187,7 +187,7 @@ public async Task HandlerPreparationFailure_DeadLettersWithoutEffectsOrSinkDeliv var sink = NewGrain(); var envelope = CreateEnvelope( receiver, - new TestMessage(HierarchicalKey.Create("preparation", "failure"), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); + new TestMessage(MessageId.Create("preparation", "failure"), 9, "preparation-failure", sink.GetGrainId(), ThrowDuringPreparation: true)); var accepted = await DeliverAsync(receiver, envelope); var state = await Fixture.WaitForDeadLetterCountAsync(receiver, 1); @@ -213,7 +213,7 @@ public async Task HandlerApplicationPreparation_AllowsIndependentWritesAndPreser var manager = context.ActivationServices.GetRequiredService(); using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/admitted-outgoing"); var envelope = CreateEnvelope(receiver, - new TestMessage(HierarchicalKey.Create("preparation", "admitted"), 91, "admitted", sink.GetGrainId()), "messages/admitted-outgoing"); + new TestMessage(MessageId.Create("preparation", "admitted"), 91, "admitted", sink.GetGrainId()), "messages/admitted-outgoing"); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); await handler.WaitUntilEnteredAsync(); var localPreparation = grain.GetSnapshotForTest(); @@ -281,7 +281,7 @@ public async Task HandlerCompletionPersistenceFailure_FreshActivationConvergesEf var oldContext = Fixture.GetGrainContext(receiver); var oldGrain = Assert.IsType(oldContext.GrainInstance); using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), "messages/completion-outcome"); - var message = new TestMessage(HierarchicalKey.Create("completion", "outcome"), 92, "completion-outcome", sink.GetGrainId()); + var message = new TestMessage(MessageId.Create("completion", "outcome"), 92, "completion-outcome", sink.GetGrainId()); var envelope = CreateEnvelope(receiver, message, "messages/completion-outcome"); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, envelope)).Status); await handler.WaitUntilEnteredAsync(); diff --git a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs index e6f1fcecdf7..1969f147cef 100644 --- a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs @@ -311,7 +311,7 @@ public async Task RecoveredOutboxEntryAddressedToItsOwnGrain_IsRejected() private MessagingTestGrain GetInstance(IMessagingTestGrain grain) => Assert.IsType(Fixture.GetGrainContext(grain).GrainInstance); - private OutboxMessage SelfMessage(IMessagingTestGrain grain, TestMessage message, string route, HierarchicalKey messageId = default) => + private OutboxMessage SelfMessage(IMessagingTestGrain grain, TestMessage message, string route, MessageId messageId = default) => TestApplicationProtocol.Create(Sessions, grain.GetGrainId(), grain.GetGrainId(), route, message, messageId); private static OutboxMessage WithDeliverAt(OutboxMessage message, DateTimeOffset? deliverAt) => diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs index 1076f11358b..d442033d3d5 100644 --- a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs +++ b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs @@ -9,7 +9,7 @@ namespace Orleans.Messaging.Tests.Support; public sealed class BootstrapDeliveryProbe : IOutgoingGrainCallFilter, ILoggerProvider { private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); - private readonly ConcurrentDictionary> _outputs = new(); + private readonly ConcurrentDictionary> _outputs = new(); private readonly ConcurrentDictionary _drains = new(); @@ -45,10 +45,10 @@ public async Task Invoke(IOutgoingGrainCallContext context) } public void Release() => _release.TrySetResult(); - public Task<(HierarchicalKey MessageId, int Value)> WaitForOutputAsync(GrainId sender) => + public Task<(MessageId MessageId, int Value)> WaitForOutputAsync(GrainId sender) => GetOutput(sender).Task.WaitAsync(TimeSpan.FromSeconds(30)); - public void OnOutput(GrainId sender, HierarchicalKey messageId, int value) => GetOutput(sender).TrySetResult((messageId, value)); - private TaskCompletionSource<(HierarchicalKey MessageId, int Value)> GetOutput(GrainId sender) => + public void OnOutput(GrainId sender, MessageId messageId, int value) => GetOutput(sender).TrySetResult((messageId, value)); + private TaskCompletionSource<(MessageId MessageId, int Value)> GetOutput(GrainId sender) => _outputs.GetOrAdd(sender, static _ => new(TaskCreationOptions.RunContinuationsAsynchronously)); } @@ -59,18 +59,18 @@ public interface IBootstrapOutputGrain : IGrainWithStringKey [GrainType("bootstrap-output")] public sealed class BootstrapOutputGrain : Grain, IBootstrapOutputGrain, IMessagingGrain, IInboxHandler, - IStateMachine, IDurableDictionaryCommandHandler + IStateMachine, IDurableDictionaryCommandHandler { - private readonly Dictionary _values = []; - private readonly List<(GrainId Sender, HierarchicalKey MessageId, int Value)> _pending = []; - private (GrainId Sender, HierarchicalKey MessageId, int Value)[] _captured = []; + private readonly Dictionary _values = []; + private readonly List<(GrainId Sender, MessageId MessageId, int Value)> _pending = []; + private (GrainId Sender, MessageId MessageId, int Value)[] _captured = []; private readonly BootstrapDeliveryProbe _probe; private readonly Orleans.Serialization.Session.SerializerSessionPool _sessions; private readonly Orleans.Serialization.Serializer _senderSerializer; - private readonly IDurableDictionaryCommandCodec _codec; + private readonly IDurableDictionaryCommandCodec _codec; public BootstrapOutputGrain(IJournaledStateManager manager, IInbox inbox, BootstrapDeliveryProbe probe, - IDurableDictionaryCommandCodec codec, Orleans.Serialization.Session.SerializerSessionPool sessions, Orleans.Serialization.Serializer senderSerializer) + IDurableDictionaryCommandCodec codec, Orleans.Serialization.Session.SerializerSessionPool sessions, Orleans.Serialization.Serializer senderSerializer) { _sessions = sessions; _senderSerializer = senderSerializer; @@ -126,8 +126,8 @@ public void Reset(JournalStreamWriter writer) public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => context.GetRequiredCommandCodec(entry.FormatKey, _codec).Apply(entry.Reader, this); - public void ApplySet(HierarchicalKey key, int value) => _values[key] = value; - public void ApplyRemove(HierarchicalKey key) => _values.Remove(key); + public void ApplySet(MessageId key, int value) => _values[key] = value; + public void ApplyRemove(MessageId key) => _values.Remove(key); public void ApplyClear() => _values.Clear(); public void Reset(int capacityHint) => _values.Clear(); } diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs index f9c744c42d4..e434a4f0b5c 100644 --- a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -83,7 +83,7 @@ public async Task SendValue(int value) { var context = Observation.Context; var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, - HierarchicalKey.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + MessageId.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), context.GrainId, OutputTarget, "output", value); Observation.Value!.Value = value; Observation.Outbox!.Send(envelope); @@ -93,7 +93,7 @@ public async Task SendSynchronousValue(int value) { var context = Observation.Context; var envelope = OutboxTestMessages.CreateApplication(context.ActivationServices, - HierarchicalKey.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), + MessageId.Create("bootstrap", context.GrainId.ToString(), "value", value.ToString(CultureInfo.InvariantCulture)), context.GrainId, OutputTarget, "output", value); Observation.Value!.Value = value; Observation.Outbox!.Send(envelope); @@ -126,7 +126,7 @@ public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationTok } var value = Observation.Value!.Value + 1; var outgoing = OutboxTestMessages.CreateApplication(Observation.Context.ActivationServices, - context.Message.Envelope.MessageId.CreateChildKey("output"), Observation.Context.GrainId, OutputTarget, "output", value); + new MessageId(context.Message.Envelope.MessageId.Key.CreateChildKey("output")), Observation.Context.GrainId, OutputTarget, "output", value); Observation.Value.Value = value; Observation.Outbox!.Send(outgoing); HandlerCalls++; @@ -292,7 +292,7 @@ public BootstrapClusterFixture(bool useServiceCollection = false) public BootstrapDeliveryProbe Delivery { get; } = new(); protected override void ConfigureServices(IServiceCollection services) { - services.AddScoped(provider => provider.GetRequiredKeyedService>("orleans-binary")); + services.AddScoped(provider => provider.GetRequiredKeyedService>("orleans-binary")); services.AddKeyedSingleton(StatelessPlacementAlias, new StatelessWorkerAttribute(1).PlacementStrategy); services.AddKeyedSingleton(OrdinaryPlacementAlias, new RandomPlacement()); services.AddSingleton(Probe); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs index 11d24f029a3..0c43675db29 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingTestGrains.cs @@ -226,7 +226,7 @@ public async Task RemoveInboxDeadLetterAsync(MessageId messageId) return true; } - public async Task RemoveOutboxDeadLetterAsync(HierarchicalKey messageId) + public async Task RemoveOutboxDeadLetterAsync(MessageId messageId) { if (!_diagnostics.RemoveOutboxDeadLetter(messageId)) { diff --git a/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs b/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs index 20aa12e5e24..a4411446cab 100644 --- a/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs +++ b/test/Orleans.Messaging.Tests/Support/OutboxStateTestAccess.cs @@ -4,8 +4,8 @@ internal sealed class OutboxStateTestAccess(IOutbox outbox) { private readonly Outbox _outbox = (Outbox)outbox; - public IEnumerable AttemptKeys => _outbox.MessageState.Keys; - public IEnumerable DeadLetterKeys => _outbox.DeadLetterState.Keys; - public OutboxMessageState Attempt(HierarchicalKey id) => _outbox.MessageState[id].State; - public OutboxDeadLetter DeadLetter(HierarchicalKey id) => _outbox.DeadLetterState[id]; + public IEnumerable AttemptKeys => _outbox.MessageState.Keys; + public IEnumerable DeadLetterKeys => _outbox.DeadLetterState.Keys; + public OutboxMessageState Attempt(MessageId id) => _outbox.MessageState[id].State; + public OutboxDeadLetter DeadLetter(MessageId id) => _outbox.DeadLetterState[id]; } diff --git a/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs index a2188e7b0ed..55e35f9431d 100644 --- a/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs +++ b/test/Orleans.Messaging.Tests/Support/OutboxTestMessages.cs @@ -7,7 +7,7 @@ namespace Orleans.Messaging.Tests.Support; internal static class OutboxTestMessages { - public static OutboxMessage Create(HierarchicalKey messageId, GrainId receiver, ReadOnlyMemory payload, + public static OutboxMessage Create(MessageId messageId, GrainId receiver, ReadOnlyMemory payload, GrainId? sender, string? subject, Serializer senderSerializer) => new() { ReceiverId = receiver, @@ -22,7 +22,7 @@ public static OutboxMessage Create(HierarchicalKey messageId, GrainId receiver, } }; - public static OutboxMessage CreateApplication(IServiceProvider services, HierarchicalKey messageId, + public static OutboxMessage CreateApplication(IServiceProvider services, MessageId messageId, GrainId sender, GrainId receiver, string subject, object? body) => Create(messageId, receiver, TestApplicationProtocol.Encode(services.GetRequiredService(), new TestApplicationMessage(subject, body)), sender, subject, services.GetRequiredService>()); diff --git a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs index fc6b0eb4174..ba9615cdc10 100644 --- a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs +++ b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs @@ -14,7 +14,7 @@ public interface IRawPayloadTestGrain : IGrainWithGuidKey } [GenerateSerializer] -public sealed record RawPayloadEffect([property: Id(0)] HierarchicalKey MessageId, [property: Id(1)] string Bytes, [property: Id(2)] int Count); +public sealed record RawPayloadEffect([property: Id(0)] MessageId MessageId, [property: Id(1)] string Bytes, [property: Id(2)] int Count); [GenerateSerializer] public sealed record RawPayloadSnapshot([property: Id(0)] Guid ActivationId, [property: Id(1)] int InboxCount, @@ -27,7 +27,7 @@ public sealed class RawPayloadTestGrain : DurableGrain, IRawPayloadTestGrain, II public const string HandlerBarrier = "raw-payload"; private readonly Inbox _inbox; private readonly IOutbox _outbox; - private readonly IDurableDictionary _effects; + private readonly IDurableDictionary _effects; private readonly IDurableValue _forward; private readonly IDurableValue _forwardCopies; internal int ExpectedAcknowledgedEffects { get; set; } = 1; @@ -37,7 +37,7 @@ public sealed class RawPayloadTestGrain : DurableGrain, IRawPayloadTestGrain, II internal Task Acknowledged => _acknowledged.Task; public RawPayloadTestGrain(IGrainContext context, IOutbox outbox, IJournaledStateManager manager, - [FromKeyedServices("raw-effects")] IDurableDictionary effects, + [FromKeyedServices("raw-effects")] IDurableDictionary effects, [FromKeyedServices("raw-forward")] IDurableValue forward, [FromKeyedServices("raw-forward-copies")] IDurableValue forwardCopies, HandlerProbe handlers) @@ -91,7 +91,7 @@ public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationTok { Envelope = index == 0 ? input - : input with { MessageId = input.MessageId.CreateChildKey(index.ToString(CultureInfo.InvariantCulture)) }, + : input with { MessageId = new MessageId(input.MessageId.Key.CreateChildKey(index.ToString(CultureInfo.InvariantCulture))) }, ReceiverId = _forward.Value }).ToArray(); _effects.TryGetValue(input.MessageId, out var prior); From 94c76c6aa9915ce290f94f33ac0c4075319faa20 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 13:10:09 -0700 Subject: [PATCH 72/95] feat(messaging): dead-letter outbox messages whose source fails When a registered source throws or returns an invalid message for a pending reference entry, the outbox dead-letters that entry alone, with its DeadLettered retirement in the same journal write, and records the delivery status source_error. The grain keeps processing its other messages. --- .../IMessagingDiagnostics.cs | 5 + src/Orleans.Messaging/Outbox.cs | 147 +++++++++-- src/Orleans.Messaging/README.md | 22 +- .../Functional/MessageSourceAndSinkTests.cs | 236 +++++++++++++++++- 4 files changed, 368 insertions(+), 42 deletions(-) diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs index 00b9c78c22c..f255507296a 100644 --- a/src/Orleans.Messaging/IMessagingDiagnostics.cs +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -18,6 +18,11 @@ public interface IMessagingDiagnostics /// /// Gets messages which could not be delivered from the outbox. /// + /// + /// A message whose failed to provide it has no materialized message: its record + /// carries only the , with an empty payload and a default + /// , and its reason records the source's failure. + /// IReadOnlyList> OutboxDeadLetters { get; } /// diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index f3a7dde8794..971eff1f8df 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -134,7 +134,8 @@ public void Send(MessageId messageId) /// /// Returns the entry's message, materializing a reference entry's message from its owning source, or returns - /// when the source has withdrawn the message. + /// when the source has withdrawn the message. Throws when the source throws or returns an + /// invalid message. /// private bool TryGetMessage(OutboxEntry entry, out OutboxMessage message) { @@ -145,8 +146,18 @@ private bool TryGetMessage(OutboxEntry entry, out OutboxMessage message) } if (entry.Source is { } owner) return TryGetSourceMessage(owner, entry.MessageId, out message); - entry.Source = FindSource(entry.MessageId, out message); - return entry.Source is not null; + + // A recovered entry's owner is the first source which doesn't decline it, including one which fails to provide + // it, so that source is notified when the entry is dead-lettered. + foreach (var source in _sources) + { + entry.Source = source; + if (TryGetSourceMessage(source, entry.MessageId, out message)) return true; + } + + entry.Source = null; + message = default; + return false; } private bool TryGetSourceMessage(IOutboxMessageSource source, MessageId messageId, out OutboxMessage message) @@ -175,6 +186,17 @@ private static void Retire(OutboxEntry entry, OutboxMessageRetirement retirement if (entry.Reference is not null) entry.Source!.OnRetired(entry.MessageId, retirement); } + /// + /// Creates the message of a dead letter whose source failed to provide it: the message ID with an empty payload and + /// a default receiver, which no valid outgoing message has. + /// + private static OutboxMessage CreateUnmaterializedMessage(MessageId messageId) => + new() { Envelope = new() { MessageId = messageId, Payload = [] }, ReceiverId = default }; + + private static bool IsUnmaterialized(OutboxMessage message) => + message.ReceiverId.IsDefault + && message.Envelope is { Payload.Length: 0, Subject: null, Headers: null, DeliverAt: null }; + private void OnRecoveryCompleted() { if (_sources.Count == 0 && _messages.Values.Any(static entry => entry.Reference is not null)) @@ -214,7 +236,15 @@ private void OnRecoveryCompleted() } foreach (var (key, letter) in _deadLetters) { - EnvelopeValidation.Validate(letter.Message); + if (IsUnmaterialized(letter.Message)) + { + EnvelopeValidation.ValidateMessageId(letter.Message.Envelope.MessageId); + } + else + { + EnvelopeValidation.Validate(letter.Message); + } + if (key != letter.Message.Envelope.MessageId) { throw new InvalidOperationException("The recovered outbox dead-letter key does not match its message identity."); @@ -256,7 +286,9 @@ private async ValueTask PersistAsync() } } - private List<(OutboxEntry Entry, OutboxMessage Message)>? SelectMessages(ref List? withdrawn) + private List<(OutboxEntry Entry, OutboxMessage Message)>? SelectMessages( + ref List? withdrawn, + ref List<(OutboxEntry Entry, Exception Error)>? failed) { var now = TimeProvider.GetUtcNow(); List<(OutboxEntry Entry, OutboxMessage Message)>? candidates = null; @@ -264,10 +296,19 @@ private async ValueTask PersistAsync() { if (entry.Sequence <= _durableSequence && IsReadyForAttempt(entry.State, now)) { - // An invalid source message fails the pass before any attempt starts. - if (!TryGetMessage(entry, out var message)) + OutboxMessage message; + try { - (withdrawn ??= []).Add(entry); + if (!TryGetMessage(entry, out message)) + { + (withdrawn ??= []).Add(entry); + continue; + } + } + catch (Exception exception) + { + // The entry is durable, so a source which can't provide a valid message dead-letters only that entry. + (failed ??= []).Add((entry, exception)); continue; } @@ -325,9 +366,11 @@ internal async ValueTask AdvanceAsync(CancellationToken cancellationToken) } List? withdrawn = null; - var candidates = SelectMessages(ref withdrawn); - if (withdrawn is not null) await RemoveWithdrawnAsync(withdrawn).ConfigureAwait(true); - if (candidates is null) return withdrawn is not null; + List<(OutboxEntry Entry, Exception Error)>? failed = null; + var candidates = SelectMessages(ref withdrawn, ref failed); + var removed = withdrawn is not null || failed is not null; + if (removed) await RemoveUnavailableAsync(withdrawn, failed).ConfigureAwait(true); + if (candidates is null) return removed; var attempts = new Task[candidates.Count]; for (var index = 0; index < attempts.Length; index++) { @@ -394,24 +437,32 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) LogDeliveryComplete(Logger, delivered, backpressured, applied - delivered - backpressured, Count); } - /// Removes reference entries whose sources withdrew their messages, without delivering or retiring them. - private async ValueTask RemoveWithdrawnAsync(List withdrawn) + /// + /// Removes reference entries whose sources withdrew their messages, without delivering or retiring them, and + /// dead-letters entries whose sources failed to provide a valid message. + /// + private async ValueTask RemoveUnavailableAsync(List? withdrawn, List<(OutboxEntry Entry, Exception Error)>? failed) { try { - foreach (var entry in withdrawn) + if (withdrawn is not null) { - Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = entry.MessageId }); - Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "withdrawn"); - if (entry.Source is null) - { - LogMessageUnprovided(Logger, entry.MessageId); - } - else + foreach (var entry in withdrawn) { - LogMessageWithdrawn(Logger, entry.MessageId); + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = entry.MessageId }); + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "withdrawn"); + if (entry.Source is null) + { + LogMessageUnprovided(Logger, entry.MessageId); + } + else + { + LogMessageWithdrawn(Logger, entry.MessageId); + } } } + + if (failed is not null) DeadLetterSourceFailures(failed); } catch (Exception exception) { @@ -421,6 +472,49 @@ private async ValueTask RemoveWithdrawnAsync(List withdrawn) await PersistAsync().ConfigureAwait(true); } + /// + /// Dead-letters reference entries whose owning sources threw or returned an invalid message, and retires them with + /// their sources in the same journal write. + /// + /// + /// No valid message exists, so each dead letter carries only the message ID; see . + /// The failed materialization counts as a delivery attempt. + /// + private void DeadLetterSourceFailures(List<(OutboxEntry Entry, Exception Error)> failed) + { + var now = TimeProvider.GetUtcNow(); + Dictionary? deadLetters = null; + foreach (var (entry, error) in failed) + { + var id = entry.MessageId; + LogSourceFailed(Logger, error, id); + var letter = new OutboxDeadLetter + { + Message = CreateUnmaterializedMessage(id), + DeadLetteredAt = now, + Reason = $"The outbox message source failed to provide the message: {error}", + AttemptCount = checked(entry.State.AttemptCount + 1) + }; + AddDeadLetter(ref deadLetters, letter); + Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id, DeadLetter = letter }); + Retire(entry, OutboxMessageRetirement.DeadLettered); + Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "source_error"); + } + + ApplyDeadLetters(deadLetters!); + } + + /// Adds a dead letter to the retained dead letters, compacting them to make room. + private void AddDeadLetter(ref Dictionary? deadLetters, OutboxDeadLetter letter) + { + var key = letter.Message.Envelope.MessageId; + deadLetters ??= _deadLetters.ToDictionary(static pair => pair.Key, static pair => pair.Value); + DeadLetterRetention.Compact(deadLetters, letter.DeadLetteredAt, Options.DeadLetterRetentionPeriod, + Options.MaxRetainedDeadLetters, static entry => entry.DeadLetteredAt, + reservedCapacity: deadLetters.ContainsKey(key) ? 0 : 1); + deadLetters[key] = letter; + } + private void ApplyDeadLetters(Dictionary entries) { foreach (var key in _deadLetters.Keys.Where(key => !entries.ContainsKey(key)).ToArray()) @@ -460,11 +554,7 @@ private void ApplyDeadLetters(Dictionary entries) Reason = error, AttemptCount = attemptCount }; - deadLetters ??= _deadLetters.ToDictionary(static pair => pair.Key, static pair => pair.Value); - DeadLetterRetention.Compact(deadLetters, now, Options.DeadLetterRetentionPeriod, - Options.MaxRetainedDeadLetters, static entry => entry.DeadLetteredAt, - reservedCapacity: deadLetters.ContainsKey(letter.Message.Envelope.MessageId) ? 0 : 1); - deadLetters[letter.Message.Envelope.MessageId] = letter; + AddDeadLetter(ref deadLetters, letter); return (null, letter); } var exponent = Math.Min(attemptCount - 1, DurableMessagingOptions.MaximumBackoffExponent); @@ -606,6 +696,9 @@ internal async Task StartAsync(CancellationToken cancellationToken) [LoggerMessage(Level = LogLevel.Warning, Message = "Removed outbox message {MessageId}, which no registered outbox message source provides")] private static partial void LogMessageUnprovided(ILogger logger, MessageId messageId); + [LoggerMessage(Level = LogLevel.Error, Message = "The outbox message source failed to provide message {MessageId}; the message is dead-lettered")] + private static partial void LogSourceFailed(ILogger logger, Exception exception, MessageId messageId); + [LoggerMessage(Level = LogLevel.Information, Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index f18a62fb78c..ad05b6af105 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -120,11 +120,16 @@ construction or activation setup, before journal recovery. `OutboxMessageRetirement.Delivered` (`Accepted` or `Duplicate`) or `DeadLettered` (receiver-reported or outbox dead-lettering) while it stages the removal, so the source's changes commit in the same journal write. An outbox dead letter stores the - materialized message. Sending a pending reference again coalesces; an ID pending as - an ordinary message, or one which no source provides, throws - `InvalidOperationException`. A source message addressed to the sending grain throws - `ArgumentException`. A recovered reference fails the activation when no source is - registered. + materialized message. After the send, a source that throws or returns an invalid + message, such as one with a different ID, dead-letters that entry alone in the + same journal write as its `DeadLettered` retirement; the grain keeps processing its + other messages. That dead letter carries only the message ID, with an empty payload + and a default receiver, and its reason records the failure. Sending a pending + reference again coalesces; an ID pending as an ordinary message, or one which no + source provides, throws `InvalidOperationException`, and an exception or invalid + message from the source during `Send` propagates without staging anything. A source + message addressed to the sending grain throws `ArgumentException`. A recovered + reference fails the activation when no source is registered. - `IInbox.Register(IInboxMessageSink)` adds a sink. After validation and deduplication, the inbox offers each message, including self-sent and scheduled messages, to the sinks in order. The first sink whose `TryAccept` returns `true` @@ -201,6 +206,7 @@ delivered messages tagged by grain type and status; histograms for processing an delivery duration; and `orleans-messaging-inbox-depth` and `orleans-messaging-outbox-depth` up-down counters for pending work. A message a grain sends to itself counts as sent and as received by its inbox, and never contributes -to outbox delivery or depth. Reference sends count as sent. A message an inbox sink -accepts counts as received `accepted` and processed `success`; one which the sink -dead-letters counts as received and processed `dead_lettered`. +to outbox delivery or depth. Reference sends count as sent; a reference entry whose +source fails to provide its message counts as delivered `source_error`. A message an +inbox sink accepts counts as received `accepted` and processed `success`; one which +the sink dead-letters counts as received and processed `dead_lettered`. diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index bdf42c7103c..0ce3dd40f1c 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -200,6 +200,180 @@ public async Task SourceMessage_WithdrawnBySource_IsRemovedWithoutDeliveryOrReti Assert.Equal(0, await sender.GetOutboxCountAsync()); } + [Theory] + [InlineData(SourceFault.Throw)] + [InlineData(SourceFault.InvalidMessage)] + [InlineData(SourceFault.MismatchedId)] + public async Task SourceFailureAfterSend_DeadLettersOnlyThatEntryAndKeepsTheGrainWorking(SourceFault fault) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var first = NewMessageId("sibling"); + var failing = NewMessageId("failing"); + var second = NewMessageId("sibling"); + await sender.StageFromSourceAsync(receiver.GetGrainId(), first, NewPayload(), write: false); + await sender.StageFromSourceAsync(receiver.GetGrainId(), failing, NewPayload(), write: false); + await sender.StageFromSourceAsync(receiver.GetGrainId(), second, NewPayload(), write: false); + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var activation)); + using var metrics = new MessageMetricListener(activation.ActivationServices.GetRequiredService().Meter); + + // The source provided every message when it was sent, and fails to provide one for its delivery attempt. + Probe.For(sender).Faults[failing] = fault; + await sender.WriteAsync(); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 3); + var (state, deadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[failing.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirement.Delivered), state.Retired[first.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirement.Delivered), state.Retired[second.ToString()]); + var received = (await OnGrainAsync(receiver, context => Grain(context).GetState())).Received; + Assert.Equal(new[] { first.ToString(), second.ToString() }.Order(), received.Keys.Order()); + AssertSourceFailureDeadLetter(Assert.Single(deadLetters), failing, fault); + + // The dead letter and its retirement commit in one write, before the write which retires the delivered siblings. + var writes = Writes(sender); + Assert.Equal(3, writes.Length); + var deadLettered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirement.DeadLettered)); + var delivered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirement.Delivered)); + Assert.True(writes[1].AsSpan().IndexOf(deadLettered) >= 0); + Assert.True(writes[1].AsSpan().IndexOf(delivered) < 0); + Assert.True(writes[2].AsSpan().IndexOf(deadLettered) < 0); + Assert.True(writes[2].AsSpan().IndexOf(delivered) >= 0); + + var grainType = sender.GetGrainId().Type.ToString(); + var outcomes = metrics.Read("outbox-messages-delivered"); + Assert.Equal(3, outcomes.Length); + Assert.Single(outcomes, row => row.Tags.Contains(new("status", "source_error"))).AssertCounter(grainType, "source_error"); + Assert.Equal(2, outcomes.Count(row => row.Tags.Contains(new("status", "accepted")))); + + // The activation keeps processing messages; the source was called once per send and once per delivery attempt. + var later = NewMessageId("later"); + await sender.StageFromSourceAsync(receiver.GetGrainId(), later, NewPayload(), write: true); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 4); + Assert.Equal(nameof(OutboxMessageRetirement.Delivered), (await OnGrainAsync(sender, context => Grain(context).GetState())).Retired[later.ToString()]); + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var current)); + Assert.Same(activation, current); + Assert.Equal(1, Probe.For(sender).Activations); + Assert.Equal(8, Probe.For(sender).Materializations); + + // The persistent fault never fails a later activation: the dead letter recovers and the source isn't called again. + await sender.DeactivateAsync(); + await activation.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + var recovered = await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters); + AssertSourceFailureDeadLetter(Assert.Single(recovered), failing, fault); + Assert.Equal(2, Probe.For(sender).Activations); + Assert.Equal(8, Probe.For(sender).Materializations); + } + + [Fact] + public async Task SourceFailureOnRecoveredEntry_DeadLettersItAndRetiresItWithTheFailingSource() + { + var grain = NewGrain(); + var receiver = NewGrain().GetGrainId(); + _ = await grain.GetOutboxCountAsync(); + var messageId = NewMessageId("recovered-failure"); + Probe.For(grain).Faults[messageId] = SourceFault.Throw; + + // Replays a reference entry whose source can no longer provide it, then runs a delivery pass. + await OnGrainAsync(grain, context => + { + Grain(context).StoreOutgoing(receiver, messageId, NewPayload()); + var outbox = Outbox(context); + ApplyReference(outbox, messageId); + ((IStateMachine)outbox).OnRecoveryCompleted(); + var pump = context.ActivationServices.GetRequiredService(); + pump.EnsureStarted(); + pump.Signal(); + return true; + }); + + await WaitUntilAsync(grain, context => Outbox(context).Count == 0); + var (state, deadLetters) = await OnGrainAsync(grain, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Empty(state.Outgoing); + Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[messageId.ToString()]); + AssertSourceFailureDeadLetter(Assert.Single(deadLetters), messageId, SourceFault.Throw); + + // The activation keeps processing messages. + var later = NewMessageId("after-recovered-failure"); + await grain.StageFromSourceAsync(receiver, later, NewPayload(), write: true); + await WaitUntilAsync(grain, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 2); + Assert.Equal(nameof(OutboxMessageRetirement.Delivered), (await OnGrainAsync(grain, context => Grain(context).GetState())).Retired[later.ToString()]); + Assert.Equal(1, Probe.For(grain).Activations); + } + + [Theory] + [InlineData(SourceFault.Throw)] + [InlineData(SourceFault.InvalidMessage)] + [InlineData(SourceFault.MismatchedId)] + public async Task SourceFailureDuringSend_ThrowsToTheCallerAndStagesNothing(SourceFault fault) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("send-failure"); + Probe.For(sender).Faults[messageId] = fault; + + var stage = sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: false); + var error = fault == SourceFault.InvalidMessage + ? await Assert.ThrowsAsync(() => stage) + : (Exception)await Assert.ThrowsAsync(() => stage); + Assert.Contains(ExpectedSourceError(fault), error.Message, StringComparison.Ordinal); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + + var later = NewMessageId("after-send-failure"); + await sender.StageFromSourceAsync(receiver.GetGrainId(), later, NewPayload(), write: true); + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + Assert.Empty(await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Equal(1, Probe.For(sender).Activations); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoveredDeadLetterWithoutReceiver_IsValidOnlyWithTheSourceFailureShape(bool withPayload) + { + var grain = NewGrain(); + _ = await grain.GetOutboxCountAsync(); + var messageId = NewMessageId("recovered-dead-letter"); + + // Replays a dead letter without a receiver, then validates the recovered state as recovery does. + var recovery = OnGrainAsync(grain, context => + { + var outbox = Outbox(context); + ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand + { + Kind = OutboxCommandKind.Delivery, + MessageId = messageId, + DeadLetter = new() + { + Message = new() { Envelope = new() { MessageId = messageId, Payload = withPayload ? NewPayload() : [] }, ReceiverId = default }, + DeadLetteredAt = DateTimeOffset.UtcNow, + Reason = "The outbox message source failed to provide the message.", + AttemptCount = 1 + } + }); + ((IStateMachine)outbox).OnRecoveryCompleted(); + return outbox.DeadLetterState.Count; + }); + + if (withPayload) + { + var error = await Assert.ThrowsAsync(() => recovery); + Assert.Contains("receiver must not be default", error.Message, StringComparison.Ordinal); + } + else + { + Assert.Equal(1, await recovery); + } + } + [Fact] public async Task RecoveredReferenceEntry_IsWithdrawnWhenNoSourceProvidesItAndFailsWithoutSources() { @@ -231,16 +405,37 @@ await OnGrainAsync(grain, context => return true; })); Assert.Contains("no outbox message source is registered", error.Message, StringComparison.Ordinal); + } - void ApplyReference(Outbox outbox, MessageId messageId) => - ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand - { - Kind = OutboxCommandKind.Enqueue, - Reference = new() { MessageId = messageId }, - State = new() { EnqueuedAt = DateTimeOffset.UtcNow } - }); + private static void ApplyReference(Outbox outbox, MessageId messageId) => + ((IDurableValueCommandHandler)outbox).ApplySet(new OutboxCommand + { + Kind = OutboxCommandKind.Enqueue, + Reference = new() { MessageId = messageId }, + State = new() { EnqueuedAt = DateTimeOffset.UtcNow } + }); + + private static void AssertSourceFailureDeadLetter(DeadLetter letter, MessageId messageId, SourceFault fault) + { + // No valid message exists, so the dead letter carries only the message ID. + Assert.Equal(messageId, letter.Message.Envelope.MessageId); + Assert.True(letter.Message.ReceiverId.IsDefault); + Assert.Empty(letter.Message.Envelope.Payload); + Assert.Null(letter.Message.Envelope.Subject); + Assert.Null(letter.Message.Envelope.Headers); + Assert.Null(letter.Message.Envelope.DeliverAt); + Assert.StartsWith("The outbox message source failed to provide the message: ", letter.Reason, StringComparison.Ordinal); + Assert.Contains(ExpectedSourceError(fault), letter.Reason, StringComparison.Ordinal); + Assert.Equal(1, letter.AttemptCount); } + private static string ExpectedSourceError(SourceFault fault) => fault switch + { + SourceFault.Throw => SourceSinkGrain.SourceFailure, + SourceFault.InvalidMessage => "The payload must not be null.", + _ => "The outbox message source returned message ID" + }; + [Fact] public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndIsOfferedAgainAfterItExpires() { @@ -543,6 +738,14 @@ public enum SourceFault MismatchedId } +/// How the source fails to provide a message which it stores. +public enum SourceFault +{ + Throw, + InvalidMessage, + MismatchedId +} + public sealed class SourceSinkProbe { private readonly ConcurrentDictionary _counters = new(); @@ -592,6 +795,7 @@ public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, { internal const string Subject = "source-sink/payload"; internal const string Rejection = "The sink rejected the message."; + internal const string SourceFailure = "The source can't decode the stored message."; private readonly IJournaledStateManager _manager; private readonly IOutbox _outbox; private readonly IDurableDictionary _outgoing; @@ -618,6 +822,7 @@ public SourceSinkGrain( _received = received; _retired = retired; _probe = probe.For(context.GrainId); + _probe.OnActivated(); outbox.Register(this); inbox.Register(this).Register(new Handler(_probe)); } @@ -625,6 +830,13 @@ public SourceSinkGrain( internal (Dictionary Outgoing, Dictionary Received, Dictionary Retired) GetState() => (new(_outgoing), new(_received), new(_retired)); + /// Stores an outgoing message in the source's state without sending it. + internal void StoreOutgoing(GrainId receiver, MessageId messageId, byte[] payload) + { + _outgoing[messageId.ToString()] = payload; + _receivers[messageId.ToString()] = receiver.ToString(); + } + public async Task StageFromSourceAsync(GrainId receiver, MessageId messageId, byte[] payload, bool write) { _outgoing[messageId.ToString()] = payload; @@ -687,6 +899,16 @@ bool IOutboxMessageSource.TryGetMessage(MessageId messageId, out OutboxMessage m Envelope = new() { MessageId = messageId, Payload = payload, Subject = Subject }, ReceiverId = GrainId.Parse(_receivers[key]) }; + if (_probe.Faults.TryGetValue(messageId, out var fault)) + { + message = fault switch + { + SourceFault.Throw => throw new InvalidOperationException(SourceFailure), + SourceFault.InvalidMessage => message with { Envelope = message.Envelope with { Payload = null! } }, + _ => message with { Envelope = message.Envelope with { MessageId = new MessageId(messageId.Key.CreateChildKey("other")) } } + }; + } + return true; } From 66e4bd021a80ee93294ab3d32b05357d3b260e27 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 15:55:13 -0700 Subject: [PATCH 73/95] feat(messaging): offer scheduled messages to inbox sinks and let sinks report delivery results A message a grain sends to itself is offered to inbox sinks whether or not it is scheduled, and a sink that backpressures it fails the Send. Covers the self-sent sink paths and the outbox source failure, recovered dead-letter, and withdrawal tests. --- src/Orleans.Messaging/IOutbox.cs | 3 +- src/Orleans.Messaging/Inbox.cs | 15 ++++++++-- .../Functional/MessageSourceAndSinkTests.cs | 30 ++++++++++++------- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/Orleans.Messaging/IOutbox.cs b/src/Orleans.Messaging/IOutbox.cs index 18b8424b4d1..53644a8ee0c 100644 --- a/src/Orleans.Messaging/IOutbox.cs +++ b/src/Orleans.Messaging/IOutbox.cs @@ -30,7 +30,8 @@ public interface IOutbox /// A message addressed to the sending grain bypasses the outbox: the grain's own inbox stages its acceptance /// for the same journal write, applying the inbox identity rules, so a completed ID is a duplicate and a different /// pending command with the ID conflicts. The inbox admits such a message regardless of its capacity and of handler - /// registration. + /// registration, and the send fails when an reports + /// for it. /// /// The envelope has an unset identity, invalid subject or destination, or exceeds identity/subject limits. /// diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 9cbc0383582..240558de6a9 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -146,12 +146,15 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel /// /// /// A synchronous send cannot report backpressure or a missing handler, so the message is admitted regardless of - /// and of handler registration. A message without + /// and of handler registration, and a sink which backpressures + /// it fails the send. A message without /// still occupies capacity for deliveries from other grains until it completes, /// and processing waits for a handler to be registered. /// /// - /// The inbox holds a different pending command with the same ID. + /// + /// The inbox holds a different pending command with the same ID, or an inbox sink backpressured the message. + /// internal void StageLocal(InboxMessage message) { if (IsDuplicate(message, out _)) @@ -160,8 +163,14 @@ internal void StageLocal(InboxMessage message) return; } - if (message.Envelope.DeliverAt is null && StageSinkAcceptance(message) is not null) + if (StageSinkAcceptance(message) is { } sinkResult) { + if (sinkResult.Status == DeliveryStatus.Backpressured) + { + throw new InvalidOperationException( + $"An inbox message sink backpressured message '{message.Envelope.MessageId}', which the grain sent to itself."); + } + return; } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 0ce3dd40f1c..ec3cddf0485 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -437,7 +437,7 @@ private static void AssertSourceFailureDeadLetter(DeadLetter lett }; [Fact] - public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndIsOfferedAgainAfterItExpires() + public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndTheSinkReportsLaterDuplicates() { var receiver = NewGrain(); var message = NewInboxMessage("deduplicated", NewPayload()); @@ -596,6 +596,24 @@ public async Task SelfSentMessage_IsAcceptedByTheSinkInTheSendingWrite() Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(grain), payload)); } + [Fact] + public async Task SelfSentMessage_FailsTheSendWhenTheSinkBackpressures() + { + var grain = NewGrain(); + _ = await grain.GetOutboxCountAsync(); + Probe.For(grain).Mode = SinkMode.Backpressure; + var writes = Writes(grain).Length; + + await Assert.ThrowsAsync( + () => grain.SendAsync(grain.GetGrainId(), NewMessageId("self-backpressured"), NewPayload(), write: true)); + + var (state, inboxCount, outboxCount) = await OnGrainAsync(grain, context => (Grain(context).GetState(), Inbox(context).Count, Outbox(context).Count)); + Assert.Empty(state.Received); + Assert.Equal(0, inboxCount); + Assert.Equal(0, outboxCount); + Assert.Equal(writes, Writes(grain).Length); + } + private ISourceSinkGrain NewGrain() => Cluster.Client.GetGrain(Guid.NewGuid()); private static MessageId NewMessageId(string name) => MessageId.Create("source-sink", name, Guid.NewGuid().ToString("N")); @@ -738,14 +756,6 @@ public enum SourceFault MismatchedId } -/// How the source fails to provide a message which it stores. -public enum SourceFault -{ - Throw, - InvalidMessage, - MismatchedId -} - public sealed class SourceSinkProbe { private readonly ConcurrentDictionary _counters = new(); @@ -920,7 +930,7 @@ void IOutboxMessageSource.OnRetired(MessageId messageId, OutboxMessageRetirement _retired[key] = retirement.ToString(); } - bool IInboxMessageSink.TryAccept(InboxMessage message) + bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result) { result = default; if (message.Envelope.Subject != Subject) return false; From 236ea89980da04fa803e1cfc1ec3cd01130c033f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 15:55:22 -0700 Subject: [PATCH 74/95] refactor(messaging): rename IInboxExtension to IMessagingExtension Updates the outbox and its tests for the renamed messaging grain extension. --- src/Orleans.Messaging/Outbox.cs | 2 +- .../Functional/JsonJournalMessagingTests.cs | 8 ++++---- .../MessagingGrainTypeConfiguratorTests.cs | 16 ++++++++-------- .../PublicInboxHandlerTransactionTests.cs | 2 +- .../Support/BootstrapOutboxServices.cs | 2 +- .../Support/RawPayloadTestGrain.cs | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 971eff1f8df..a1ab1e25600 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -335,7 +335,7 @@ private async Task DeliverAsync(OutboxEntry entry, OutboxMessag var start = Stopwatch.GetTimestamp(); try { - var target = Shared.GrainFactory.GetGrain(message.ReceiverId); + var target = Shared.GrainFactory.GetGrain(message.ReceiverId); var result = await target.DeliverAsync(message.ToInboxMessage(), cancellationToken).ConfigureAwait(true); return new(entry, message, result, null, Stopwatch.GetElapsedTime(start), false); } diff --git a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs index f33bfbd1b9d..a6c6ff7a876 100644 --- a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs @@ -42,7 +42,7 @@ public async Task JsonJournal_RecoversOutboxAndInboxStateAcrossActivations() var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); var redelivery = type.Create(messageId, receiver.GetGrainId(), 42).ToInboxMessage(); - Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync(redelivery, Token)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync(redelivery, Token)).Status); Assert.Equal(new[] { 42 }, Probe.Get(receiver.GetGrainId())); } @@ -55,7 +55,7 @@ public async Task CustomInboxHandlers_FirstHandlerWhichCanHandleReceivesMessage( Envelope = new Envelope { MessageId = MessageId.Create("custom", Guid.NewGuid().ToString("N")), Payload = [7], Subject = "custom" } }; - var result = await receiver.AsReference().DeliverAsync(message, Token); + var result = await receiver.AsReference().DeliverAsync(message, Token); Assert.Equal(DeliveryStatus.Accepted, result.Status); Assert.Equal(new[] { CustomHandlerGrain.FirstMatch }, await Probe.WaitAsync(receiver.GetGrainId(), 1, Token)); @@ -69,7 +69,7 @@ public async Task JsonJournal_RecoversScheduledInboxMessageWithItsDeliverAt() var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); var deliverAt = new DateTimeOffset(DateTime.UtcNow.AddDays(1).Ticks, TimeSpan.Zero).ToOffset(TimeSpan.FromHours(5)); var scheduled = type.Create(messageId, receiver.GetGrainId(), 7, deliverAt).ToInboxMessage(); - var extension = receiver.AsReference(); + var extension = receiver.AsReference(); Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(scheduled, Token)).Status); Assert.True(Cluster.TryGetGrainContext(receiver.GetGrainId(), out var receiverContext)); @@ -116,7 +116,7 @@ public async Task JsonJournal_RecoversSelfSentMessagesFromTheInbox() Assert.Equal((1, deliverAt), await recovered.Task.WaitAsync(TimeSpan.FromSeconds(30), Token)); var type = new MessageType(JsonMessagingGrain.Subject, Cluster.Client.ServiceProvider.GetRequiredService>()); var redelivery = type.Create(immediateId, grain.GetGrainId(), 8).ToInboxMessage(); - Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(redelivery, Token)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(redelivery, Token)).Status); Assert.Equal(new[] { 8 }, Probe.Get(grain.GetGrainId())); } diff --git a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs index b9853aadf48..87a944df5ef 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessagingGrainTypeConfiguratorTests.cs @@ -39,7 +39,7 @@ protected MessagingGrainTypeConfiguratorTests(BootstrapClusterFixture fixture) : [InlineData(typeof(GenericBootstrapGrain))] [InlineData(typeof(DurableBootstrapGrain))] [InlineData(typeof(MarkedDurableBootstrapGrain))] - public async Task SelectedComposition_InstallsInboxExtensionAndReplaysFreshScopedState(Type grainClass) + public async Task SelectedComposition_InstallsMessagingExtensionAndReplaysFreshScopedState(Type grainClass) { var grain = CreateGrain(grainClass); Assert.Equal(0, await grain.GetValueAsync()); @@ -53,7 +53,7 @@ public async Task SelectedComposition_InstallsInboxExtensionAndReplaysFreshScope using var handler = Fixture.HandlerProbe.Arm(grain.GetGrainId(), BootstrapState.Route); var envelope = CreateEnvelope(grain); - Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(DeliveryStatus.Accepted, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); await handler.WaitUntilEnteredAsync(); Assert.Equal(41, first.Value!.Value); Assert.Equal(1, first.Inbox!.Count); @@ -66,7 +66,7 @@ public async Task SelectedComposition_InstallsInboxExtensionAndReplaysFreshScope Assert.Equal(1, GetProcessedCount(first)); var output = Assert.Single(first.Outbox!.Messages()); Assert.NotEmpty(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, grain.GetGrainId())); - Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); Assert.Equal(1, state.HandlerCalls); var other = CreateGrain(grainClass); @@ -96,7 +96,7 @@ public async Task SelectedComposition_InstallsInboxExtensionAndReplaysFreshScope Assert.Same(setup, GetSetup(recovered.Context)); Assert.Equal(output.Envelope.MessageId, Assert.Single(recovered.Outbox!.Messages()).Envelope.MessageId); Assert.Equal(1, GetProcessedCount(recovered)); - Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await grain.AsReference().DeliverAsync(envelope, Cancellation)).Status); Assert.Equal(0, recoveredState.HandlerCalls); Assert.Equal(2, Fixture.Storage.GetReadCount(journal)); @@ -110,7 +110,7 @@ public async Task SelectedComposition_InstallsInboxExtensionAndReplaysFreshScope Assert.Empty(recovered.Outbox!.Messages()); var sink = Fixture.Client.GetGrain("capture"); Assert.Equal(1, await sink.GetMessageCountAsync()); - Assert.Equal(DeliveryStatus.Duplicate, (await sink.AsReference().DeliverAsync(output.ToInboxMessage(), Cancellation)).Status); + Assert.Equal(DeliveryStatus.Duplicate, (await sink.AsReference().DeliverAsync(output.ToInboxMessage(), Cancellation)).Status); Assert.Equal(1, await sink.GetMessageCountAsync()); } @@ -269,10 +269,10 @@ public async Task ReentrantAndInterleavingModels_AreAcceptedAndBindTheInbox(Type var observation = Assert.Single(Probe.Get(grain.GetGrainId())); Assert.Equal(1, observation.Activations); var inbox = observation.Context.ActivationServices.GetRequiredService(); - Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); + Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); // No handler is registered by control grains, so the installed extension reports that explicitly. - var result = await grain.AsReference().DeliverAsync(CreateEnvelope(grain), Cancellation); + var result = await grain.AsReference().DeliverAsync(CreateEnvelope(grain), Cancellation); Assert.Equal(DeliveryStatus.HandlerNotFound, result.Status); Assert.False(observation.Context.Deactivated.IsCompleted); } @@ -446,7 +446,7 @@ private static void AssertComposition(BootstrapObservation observation, Bootstra Assert.Same(inbox, inboxState); Assert.True(observation.Manager.TryGetStateMachine(MessagingStateNames.Outbox, out var outboxState)); Assert.Same(outbox, outboxState); - Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); + Assert.Same(inbox, observation.Context.GetComponent()!.GetExtension()); Assert.Same(state, services.GetRequiredService()); } diff --git a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs index 628435b84c9..1a7bf3b986e 100644 --- a/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/PublicInboxHandlerTransactionTests.cs @@ -114,7 +114,7 @@ public async Task ArbitraryRawPayload_AcceptanceReplayForwardAndBusinessCompleti Assert.Equal(acknowledged.Effects, recovered.Effects); Assert.Equal(1, recovered.ProcessedCount); Assert.Equal(0, recovered.OutboxCount); - Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync( + Assert.Equal(DeliveryStatus.Duplicate, (await receiver.AsReference().DeliverAsync( new InboxMessage { Envelope = envelope }, TestContext.Current.CancellationToken)).Status); Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); var sinkState = await sink.GetSnapshotAsync(); diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs index d442033d3d5..b407d72738e 100644 --- a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs +++ b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs @@ -37,7 +37,7 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except public async Task Invoke(IOutgoingGrainCallContext context) { - if (context.TargetId == BootstrapState.OutputTarget && context.MethodName == nameof(IInboxExtension.DeliverAsync)) + if (context.TargetId == BootstrapState.OutputTarget && context.MethodName == nameof(IMessagingExtension.DeliverAsync)) { await _release.Task.WaitAsync(context.Request.GetCancellationToken()); } diff --git a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs index ba9615cdc10..f97fc38b22a 100644 --- a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs +++ b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs @@ -67,7 +67,7 @@ public async Task ConfigureForwardAsync(GrainId target, int copies = 1) public async Task AcceptAndDeactivateAsync(InboxMessage message) { - var result = await ((IInboxExtension)_inbox).DeliverAsync(message); + var result = await ((IMessagingExtension)_inbox).DeliverAsync(message); AcceptedSnapshot = GetSnapshotForTest(); DeactivateOnIdle(); return result; From 7c1feef666be97c254dff17b41a591dc0e48a125 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 16:08:48 -0700 Subject: [PATCH 75/95] feat(messaging): run outbox delivery attempts through a Polly resilience pipeline Each outbox call to a receiver runs through DurableMessagingOptions.DeliveryResiliencePipeline. When it is null, AddMessaging's default pipeline bounds the attempt at six response timeouts, retries failed calls five times with jittered exponential backoff, and opens a silo-wide circuit breaker on correlated failures. Calls which fail through the pipeline retry durably after the maximum backoff without consuming MaxDeliveryAttempts, so infrastructure outages shorter than MaxOutboxRetryAge delay messages instead of dead-lettering them. --- .../DefaultDeliveryResiliencePipeline.cs | 58 +++++++ .../Hosting/MessagingExtensions.cs | 1 + src/Orleans.Messaging/MessagingShared.cs | 5 + src/Orleans.Messaging/Outbox.cs | 34 +++- src/Orleans.Messaging/README.md | 12 +- .../OutboxDeliveryResilienceTests.cs | 156 ++++++++++++++++++ 6 files changed, 258 insertions(+), 8 deletions(-) create mode 100644 src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs create mode 100644 test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs diff --git a/src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs b/src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs new file mode 100644 index 00000000000..98e996f3bf4 --- /dev/null +++ b/src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs @@ -0,0 +1,58 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.DurableJobs; +using Polly; +using Polly.CircuitBreaker; +using Polly.Retry; + +namespace Orleans.Messaging; + +/// +/// Holds the silo-wide outbox delivery pipeline used when +/// is . +/// +/// +/// Orleans fails each receiver call after the response timeout, so the pipeline bounds retries and the whole attempt +/// in multiples of that timeout. Its circuit breaker is shared by every outbox in the silo, so it opens on failures +/// correlated across receivers, such as a storage outage. Delays use the durable jobs clock, which also times the +/// outbox's durable retries. +/// +internal sealed class DefaultDeliveryResiliencePipeline( + IOptions messagingOptions, + [FromKeyedServices(DurableJobTimeProviderNames.DurableJobs)] TimeProvider timeProvider) +{ + private const int MaxRetryAttempts = 5; + + public ResiliencePipeline Pipeline { get; } = Create(messagingOptions.Value.ResponseTimeout, timeProvider); + + private static ResiliencePipeline Create(TimeSpan responseTimeout, TimeProvider timeProvider) + { + var maxTimeout = TimeSpan.FromDays(1); + var responseTimeoutOrMax = responseTimeout > TimeSpan.Zero && responseTimeout < maxTimeout ? responseTimeout : maxTimeout; + var shouldHandle = new PredicateBuilder().Handle(static exception => exception is not (OperationCanceledException or BrokenCircuitException)); + return new ResiliencePipelineBuilder { TimeProvider = timeProvider } + .AddTimeout(Clamp(responseTimeoutOrMax * (MaxRetryAttempts + 1), TimeSpan.FromSeconds(1), maxTimeout)) + .AddRetry(new RetryStrategyOptions + { + ShouldHandle = shouldHandle, + MaxRetryAttempts = MaxRetryAttempts, + Delay = TimeSpan.FromSeconds(1), + MaxDelay = responseTimeoutOrMax, + BackoffType = DelayBackoffType.Exponential, + UseJitter = true, + }) + .AddCircuitBreaker(new CircuitBreakerStrategyOptions + { + ShouldHandle = shouldHandle, + FailureRatio = 0.5, + MinimumThroughput = 20, + SamplingDuration = TimeSpan.FromSeconds(30), + BreakDuration = Clamp(responseTimeoutOrMax, TimeSpan.FromSeconds(1), maxTimeout), + }) + .Build(); + } + + private static TimeSpan Clamp(TimeSpan value, TimeSpan min, TimeSpan max) => value < min ? min : value > max ? max : value; +} diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs index f8ce5b3d1eb..f4ecde01cf2 100644 --- a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -75,6 +75,7 @@ public static IServiceCollection AddMessaging(this IServiceCollection services, services.ConfigureNamedOptionForLogging(Options.DefaultName); services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddScoped(); services.TryAddScoped(); services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); diff --git a/src/Orleans.Messaging/MessagingShared.cs b/src/Orleans.Messaging/MessagingShared.cs index 8362460eb56..e4be77ef91b 100644 --- a/src/Orleans.Messaging/MessagingShared.cs +++ b/src/Orleans.Messaging/MessagingShared.cs @@ -6,6 +6,7 @@ using Orleans.Journaling; using Orleans.Messaging.Configuration; using Orleans.Runtime; +using Polly; namespace Orleans.Messaging; @@ -22,6 +23,7 @@ public MessagingShared(IServiceProvider services, GrainType grainType) { GrainType = grainType.ToString(); Options = services.GetRequiredService>().Value; + DeliveryPipeline = Options.DeliveryResiliencePipeline ?? services.GetRequiredService().Pipeline; TimeProvider = services.GetRequiredService(); JobTimeProvider = services.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs); JobManager = services.GetRequiredService(); @@ -41,6 +43,9 @@ public MessagingShared(IServiceProvider services, GrainType grainType) public DurableMessagingOptions Options { get; } + /// Gets the pipeline which runs each outbox delivery attempt. + public ResiliencePipeline DeliveryPipeline { get; } + public TimeProvider TimeProvider { get; } /// Gets the durable jobs clock, which defines the time domain of wake-up due times. diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index a1ab1e25600..9026f9eeccf 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -10,6 +10,7 @@ using Orleans.Messaging.Configuration; using Orleans.Journaling; using Orleans.Runtime; +using Polly; namespace Orleans.Messaging; @@ -336,7 +337,30 @@ private async Task DeliverAsync(OutboxEntry entry, OutboxMessag try { var target = Shared.GrainFactory.GetGrain(message.ReceiverId); - var result = await target.DeliverAsync(message.ToInboxMessage(), cancellationToken).ConfigureAwait(true); + var inboxMessage = message.ToInboxMessage(); + var pipeline = Shared.DeliveryPipeline; + DeliveryResult result; + if (ReferenceEquals(pipeline, ResiliencePipeline.Empty)) + { + result = await target.DeliverAsync(inboxMessage, cancellationToken).ConfigureAwait(true); + } + else + { + // Continue on the activation's scheduler so each try calls the receiver from this grain. + var context = ResilienceContextPool.Shared.Get(continueOnCapturedContext: true, cancellationToken); + try + { + result = await pipeline.ExecuteAsync( + static (context, state) => state.target.DeliverAsync(state.inboxMessage, context.CancellationToken), + context, + (target, inboxMessage)).ConfigureAwait(true); + } + finally + { + ResilienceContextPool.Shared.Return(context); + } + } + return new(entry, message, result, null, Stopwatch.GetElapsedTime(start), false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -537,14 +561,16 @@ private void ApplyDeadLetters(Dictionary entries) var previous = outcome.Entry.State; var status = outcome.Result?.Status; var now = TimeProvider.GetUtcNow(); - var attemptCount = outcome.Expired ? previous.AttemptCount : checked(previous.AttemptCount + 1); + // Receivers answer counted attempts; failed calls, such as those during an outage, retry until the message expires. + var answered = outcome.Result is not null; + var attemptCount = answered ? checked(previous.AttemptCount + 1) : previous.AttemptCount; var error = outcome.Error ?? status switch { DeliveryStatus.Backpressured => "The receiver is backpressured.", DeliveryStatus.HandlerNotFound => outcome.Result?.Message ?? "The receiver has no registered inbox handler.", _ => $"Unexpected delivery status '{status}'." }; - if (outcome.Expired || attemptCount >= Options.MaxDeliveryAttempts + if (outcome.Expired || answered && attemptCount >= Options.MaxDeliveryAttempts || MessagingTime.IsExpired(now, previous.EnqueuedAt!.Value, Options.MaxOutboxRetryAge)) { var letter = new OutboxDeadLetter @@ -557,7 +583,7 @@ private void ApplyDeadLetters(Dictionary entries) AddDeadLetter(ref deadLetters, letter); return (null, letter); } - var exponent = Math.Min(attemptCount - 1, DurableMessagingOptions.MaximumBackoffExponent); + var exponent = answered ? Math.Min(attemptCount - 1, DurableMessagingOptions.MaximumBackoffExponent) : DurableMessagingOptions.MaximumBackoffExponent; return (new OutboxMessageState { AttemptCount = attemptCount, diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index ad05b6af105..07d289a29cb 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -174,10 +174,14 @@ write completes. Each receiver returns a `DeliveryResult`: - `DeadLettered`: the receiver moved the message to its dead-letter store. `Accepted`, `Duplicate`, and `DeadLettered` are terminal: the outbox removes the -message. The outbox retries `Backpressured`, `HandlerNotFound`, and transport -failures with exponential backoff. It dead-letters a message after -`MaxDeliveryAttempts` or once it is older than `MaxOutboxRetryAge`. Delivery order -is unspecified. +message. Each delivery attempt runs through the `DeliveryResiliencePipeline` Polly +pipeline. By default, it retries failed calls five times with exponential backoff +within a total timeout of six response timeouts, and a silo-wide circuit breaker +opens on correlated failures. The outbox retries `Backpressured`, `HandlerNotFound`, +and calls which fail through the pipeline with exponential backoff. It dead-letters +a message after `MaxDeliveryAttempts` replies without acceptance, or once it is +older than `MaxOutboxRetryAge`, so failed calls during an outage retry until that +age. Delivery order is unspecified. Each activation runs one messaging pump on its scheduler, processing inbox batches and delivering outbox batches. One durable job per grain wakes the pump while either diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs new file mode 100644 index 00000000000..827255bdddd --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs @@ -0,0 +1,156 @@ +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Polly; +using Polly.CircuitBreaker; +using Polly.Retry; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class OutboxDeliveryResilienceTests +{ + private const int InPipelineRetries = 2; + + [Fact] + public async Task FailedTry_IsRetriedWithinTheDeliveryAttempt() + { + await using var fixture = await StartAsync(static strategy => new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + ShouldHandle = new PredicateBuilder().Handle(), + MaxRetryAttempts = InPipelineRetries, + Delay = TimeSpan.Zero, + }) + .AddStrategy(_ => strategy, new FaultStrategyOptions()) + .Build()); + fixture.Strategy.FailNext(InPipelineRetries); + + var (sender, receiver) = await SendAsync(fixture, "retried-in-attempt"); + + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("retried-in-attempt", Assert.Single(delivered.Effects).Value); + Assert.Equal(InPipelineRetries + 1, fixture.Strategy.Executions); + var drained = await fixture.WaitForOutboxCountAsync(sender, 0); + Assert.Empty(drained.OutboxDeadLetters); + } + + [Fact] + public async Task FailuresLeavingThePipeline_RetryDurablyWithoutConsumingDeliveryAttempts() + { + await using var fixture = await StartAsync(static strategy => new ResiliencePipelineBuilder() + .AddStrategy(_ => strategy, new FaultStrategyOptions()) + .Build()); + const int FailedAttempts = 3; + fixture.Strategy.FailNext(FailedAttempts); + + var (sender, receiver) = await SendAsync(fixture, "retried-durably"); + + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("retried-durably", Assert.Single(delivered.Effects).Value); + Assert.Equal(FailedAttempts + 1, fixture.Strategy.Executions); + var drained = await fixture.WaitForOutboxCountAsync(sender, 0); + Assert.Empty(drained.OutboxDeadLetters); + } + + [Fact] + public async Task EmptyPipeline_DeliversDirectly() + { + await using var fixture = await StartAsync(static _ => ResiliencePipeline.Empty); + + var (sender, receiver) = await SendAsync(fixture, "direct"); + + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("direct", Assert.Single(delivered.Effects).Value); + Assert.Equal(0, fixture.Strategy.Executions); + Assert.Empty((await fixture.WaitForOutboxCountAsync(sender, 0)).OutboxDeadLetters); + } + + [Fact] + public async Task DefaultPipeline_RetriesFailedCallsAndPropagatesOpenCircuit() + { + Assert.Null(new DurableMessagingOptions().DeliveryResiliencePipeline); + var pipeline = new DefaultDeliveryResiliencePipeline( + Options.Create(new SiloMessagingOptions { ResponseTimeout = TimeSpan.FromMilliseconds(100) }), + TimeProvider.System).Pipeline; + var cancellationToken = TestContext.Current.CancellationToken; + + var calls = 0; + var result = await pipeline.ExecuteAsync( + _ => ++calls == 1 ? throw new IOException("Injected outage.") : ValueTask.FromResult(DeliveryResult.Accepted()), + cancellationToken); + Assert.Equal(DeliveryStatus.Accepted, result.Status); + Assert.Equal(2, calls); + + calls = 0; + await Assert.ThrowsAsync(() => pipeline.ExecuteAsync( + _ => + { + calls++; + throw new BrokenCircuitException(); + }, + cancellationToken).AsTask()); + Assert.Equal(1, calls); + } + + private static async Task StartAsync(Func createPipeline) + { + var fixture = new DeliveryResilienceClusterFixture(createPipeline); + await fixture.InitializeAsync(); + return fixture; + } + + private static async Task<(IMessagingTestGrain Sender, IMessagingTestGrain Receiver)> SendAsync(MessagingClusterFixture fixture, string value) + { + var sender = fixture.Client.GetGrain(Guid.NewGuid()); + var receiver = fixture.Client.GetGrain(Guid.NewGuid()); + await sender.SendAsync( + receiver.GetGrainId(), + "messages/resilience", + new TestMessage(TestApplicationProtocol.NewMessageId(), 1, value)); + return (sender, receiver); + } + + private sealed class DeliveryResilienceClusterFixture(Func createPipeline) + : MessagingClusterFixture(initialSilos: 1) + { + public FaultStrategy Strategy { get; } = new(); + + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxDeliveryAttempts = 1; + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(5); + options.DeliveryResiliencePipeline = createPipeline(Strategy); + } + } + + /// Fails a configured number of tries before calling the receiver. + private sealed class FaultStrategy : ResilienceStrategy + { + private int _failures; + private int _executions; + + public int Executions => Volatile.Read(ref _executions); + + public void FailNext(int count) => Volatile.Write(ref _failures, count); + + protected override ValueTask> ExecuteCore( + Func>> callback, + ResilienceContext context, + TState state) + { + Interlocked.Increment(ref _executions); + return Interlocked.Decrement(ref _failures) >= 0 + ? ValueTask.FromResult(Outcome.FromException(new TimeoutException("Injected delivery failure."))) + : callback(context, state); + } + } + + private sealed class FaultStrategyOptions : ResilienceStrategyOptions; +} From bf97fd3c3218b16f7ead239c4a8204ce7caaa297 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 17:59:22 -0700 Subject: [PATCH 76/95] feat(messaging)!: demote failed deliveries to later batches and retry backpressure indefinitely Remove the default delivery resilience pipeline and its shared circuit breaker; DeliveryResiliencePipeline defaults to ResiliencePipeline.Empty. Failed attempts reschedule durably with exponential backoff while other messages continue. Backpressured retries until the receiver takes the message. HandlerNotFound logs an error and dead-letters after MaxHandlerNotFoundRetryAge (6 hours). Unanswered attempts dead-letter after MaxOutboxRetryAge, which restarts on each receiver answer. MaxDeliveryAttempts is removed. --- .../DefaultDeliveryResiliencePipeline.cs | 58 ------ .../Hosting/MessagingExtensions.cs | 1 - src/Orleans.Messaging/MessageState.cs | 7 +- src/Orleans.Messaging/MessagingShared.cs | 5 - src/Orleans.Messaging/Outbox.cs | 71 ++++--- src/Orleans.Messaging/README.md | 16 +- .../MessagingCommandSerializationTests.cs | 16 +- .../Functional/MessageSourceAndSinkTests.cs | 8 +- .../OutboxDeliveryResilienceTests.cs | 195 +++++++++++++----- ...ublicMessagingRoutingAndDeadLetterTests.cs | 2 +- .../Functional/SelfAddressedMessageTests.cs | 2 +- 11 files changed, 222 insertions(+), 159 deletions(-) delete mode 100644 src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs diff --git a/src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs b/src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs deleted file mode 100644 index 98e996f3bf4..00000000000 --- a/src/Orleans.Messaging/DefaultDeliveryResiliencePipeline.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using Orleans.Configuration; -using Orleans.DurableJobs; -using Polly; -using Polly.CircuitBreaker; -using Polly.Retry; - -namespace Orleans.Messaging; - -/// -/// Holds the silo-wide outbox delivery pipeline used when -/// is . -/// -/// -/// Orleans fails each receiver call after the response timeout, so the pipeline bounds retries and the whole attempt -/// in multiples of that timeout. Its circuit breaker is shared by every outbox in the silo, so it opens on failures -/// correlated across receivers, such as a storage outage. Delays use the durable jobs clock, which also times the -/// outbox's durable retries. -/// -internal sealed class DefaultDeliveryResiliencePipeline( - IOptions messagingOptions, - [FromKeyedServices(DurableJobTimeProviderNames.DurableJobs)] TimeProvider timeProvider) -{ - private const int MaxRetryAttempts = 5; - - public ResiliencePipeline Pipeline { get; } = Create(messagingOptions.Value.ResponseTimeout, timeProvider); - - private static ResiliencePipeline Create(TimeSpan responseTimeout, TimeProvider timeProvider) - { - var maxTimeout = TimeSpan.FromDays(1); - var responseTimeoutOrMax = responseTimeout > TimeSpan.Zero && responseTimeout < maxTimeout ? responseTimeout : maxTimeout; - var shouldHandle = new PredicateBuilder().Handle(static exception => exception is not (OperationCanceledException or BrokenCircuitException)); - return new ResiliencePipelineBuilder { TimeProvider = timeProvider } - .AddTimeout(Clamp(responseTimeoutOrMax * (MaxRetryAttempts + 1), TimeSpan.FromSeconds(1), maxTimeout)) - .AddRetry(new RetryStrategyOptions - { - ShouldHandle = shouldHandle, - MaxRetryAttempts = MaxRetryAttempts, - Delay = TimeSpan.FromSeconds(1), - MaxDelay = responseTimeoutOrMax, - BackoffType = DelayBackoffType.Exponential, - UseJitter = true, - }) - .AddCircuitBreaker(new CircuitBreakerStrategyOptions - { - ShouldHandle = shouldHandle, - FailureRatio = 0.5, - MinimumThroughput = 20, - SamplingDuration = TimeSpan.FromSeconds(30), - BreakDuration = Clamp(responseTimeoutOrMax, TimeSpan.FromSeconds(1), maxTimeout), - }) - .Build(); - } - - private static TimeSpan Clamp(TimeSpan value, TimeSpan min, TimeSpan max) => value < min ? min : value > max ? max : value; -} diff --git a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs index f4ecde01cf2..f8ce5b3d1eb 100644 --- a/src/Orleans.Messaging/Hosting/MessagingExtensions.cs +++ b/src/Orleans.Messaging/Hosting/MessagingExtensions.cs @@ -75,7 +75,6 @@ public static IServiceCollection AddMessaging(this IServiceCollection services, services.ConfigureNamedOptionForLogging(Options.DefaultName); services.TryAddSingleton(); - services.TryAddSingleton(); services.TryAddScoped(); services.TryAddScoped(); services.Replace(ServiceDescriptor.Scoped(static sp => sp.GetRequiredService())); diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs index c80ec0b2adb..7c75bba4986 100644 --- a/src/Orleans.Messaging/MessageState.cs +++ b/src/Orleans.Messaging/MessageState.cs @@ -35,8 +35,13 @@ internal sealed class OutboxMessageState [Id(2)] public string? LastError { get; set; } + /// Gets or sets when the first delivery attempt without a receiver answer started, since the receiver last answered. [Id(3)] - public DateTimeOffset? EnqueuedAt { get; set; } + public DateTimeOffset? UnansweredSince { get; set; } + + /// Gets or sets when the receiver first reported no inbox handler, since it last answered otherwise. + [Id(4)] + public DateTimeOffset? HandlerNotFoundSince { get; set; } } [GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] diff --git a/src/Orleans.Messaging/MessagingShared.cs b/src/Orleans.Messaging/MessagingShared.cs index e4be77ef91b..8362460eb56 100644 --- a/src/Orleans.Messaging/MessagingShared.cs +++ b/src/Orleans.Messaging/MessagingShared.cs @@ -6,7 +6,6 @@ using Orleans.Journaling; using Orleans.Messaging.Configuration; using Orleans.Runtime; -using Polly; namespace Orleans.Messaging; @@ -23,7 +22,6 @@ public MessagingShared(IServiceProvider services, GrainType grainType) { GrainType = grainType.ToString(); Options = services.GetRequiredService>().Value; - DeliveryPipeline = Options.DeliveryResiliencePipeline ?? services.GetRequiredService().Pipeline; TimeProvider = services.GetRequiredService(); JobTimeProvider = services.GetRequiredKeyedService(DurableJobTimeProviderNames.DurableJobs); JobManager = services.GetRequiredService(); @@ -43,9 +41,6 @@ public MessagingShared(IServiceProvider services, GrainType grainType) public DurableMessagingOptions Options { get; } - /// Gets the pipeline which runs each outbox delivery attempt. - public ResiliencePipeline DeliveryPipeline { get; } - public TimeProvider TimeProvider { get; } /// Gets the durable jobs clock, which defines the time domain of wake-up due times. diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 9026f9eeccf..37a553007ad 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -75,7 +75,7 @@ public void Send(OutboxMessage message) try { - Emit(new() { Kind = OutboxCommandKind.Enqueue, Message = message, State = new() { EnqueuedAt = TimeProvider.GetUtcNow() } }); + Emit(new() { Kind = OutboxCommandKind.Enqueue, Message = message, State = new() }); Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); } catch (Exception exception) @@ -110,7 +110,7 @@ public void Send(MessageId messageId) { Kind = OutboxCommandKind.Enqueue, Reference = new() { MessageId = messageId }, - State = new() { EnqueuedAt = TimeProvider.GetUtcNow() } + State = new() }); _messages[messageId].Source = source; Shared.Instruments.OnOutboxMessageSent(Shared.GrainType); @@ -222,10 +222,6 @@ private void OnRecoveryCompleted() EnvelopeValidation.ValidateMessageId(key); } - if (entry.State.EnqueuedAt is null) - { - throw new InvalidOperationException("A recovered outbox message requires its enqueue time."); - } if (key != entry.MessageId) { throw new InvalidOperationException("The recovered outbox key does not match its command identity."); @@ -328,17 +324,17 @@ private async ValueTask PersistAsync() private async Task DeliverAsync(OutboxEntry entry, OutboxMessage message, CancellationToken cancellationToken) { - if (entry.State.EnqueuedAt is { } enqueuedAt - && MessagingTime.IsExpired(TimeProvider.GetUtcNow(), enqueuedAt, Options.MaxOutboxRetryAge)) + var startedAt = TimeProvider.GetUtcNow(); + if (IsRetryExpired(entry.State, startedAt)) { - return new(entry, message, null, $"The message exceeded the maximum retry age of {Options.MaxOutboxRetryAge}.", TimeSpan.Zero, true); + return new(entry, message, null, $"The receiver did not answer delivery attempts within {Options.MaxOutboxRetryAge}.", TimeSpan.Zero, startedAt, true); } var start = Stopwatch.GetTimestamp(); try { var target = Shared.GrainFactory.GetGrain(message.ReceiverId); var inboxMessage = message.ToInboxMessage(); - var pipeline = Shared.DeliveryPipeline; + var pipeline = Options.DeliveryResiliencePipeline; DeliveryResult result; if (ReferenceEquals(pipeline, ResiliencePipeline.Empty)) { @@ -361,11 +357,11 @@ private async Task DeliverAsync(OutboxEntry entry, OutboxMessag } } - return new(entry, message, result, null, Stopwatch.GetElapsedTime(start), false); + return new(entry, message, result, null, Stopwatch.GetElapsedTime(start), startedAt, false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - return new(entry, message, null, "Delivery was canceled.", Stopwatch.GetElapsedTime(start), false); + return new(entry, message, null, "Delivery was canceled.", Stopwatch.GetElapsedTime(start), startedAt, false); } catch (Exception exception) { @@ -373,7 +369,7 @@ private async Task DeliverAsync(OutboxEntry entry, OutboxMessag { LogDeliveryError(Logger, exception, message.Envelope.MessageId, message.ReceiverId); } - return new(entry, message, null, exception.ToString(), Stopwatch.GetElapsedTime(start), false); + return new(entry, message, null, exception.ToString(), Stopwatch.GetElapsedTime(start), startedAt, false); } } @@ -561,17 +557,28 @@ private void ApplyDeadLetters(Dictionary entries) var previous = outcome.Entry.State; var status = outcome.Result?.Status; var now = TimeProvider.GetUtcNow(); - // Receivers answer counted attempts; failed calls, such as those during an outage, retry until the message expires. - var answered = outcome.Result is not null; - var attemptCount = answered ? checked(previous.AttemptCount + 1) : previous.AttemptCount; + var attemptCount = outcome.Expired ? previous.AttemptCount : checked(previous.AttemptCount + 1); + + // A receiver checks its deduplication records before it answers, so an answer proves that no earlier attempt was + // accepted. Only attempts since the receiver last answered can have been accepted, which bounds their retry by + // MaxOutboxRetryAge, within the receiver's deduplication window. + var unansweredSince = outcome.Result is null ? previous.UnansweredSince ?? outcome.StartedAt : (DateTimeOffset?)null; + var handlerNotFoundSince = status switch + { + null => previous.HandlerNotFoundSince, + DeliveryStatus.HandlerNotFound => previous.HandlerNotFoundSince ?? outcome.StartedAt, + _ => null + }; var error = outcome.Error ?? status switch { DeliveryStatus.Backpressured => "The receiver is backpressured.", DeliveryStatus.HandlerNotFound => outcome.Result?.Message ?? "The receiver has no registered inbox handler.", _ => $"Unexpected delivery status '{status}'." }; - if (outcome.Expired || answered && attemptCount >= Options.MaxDeliveryAttempts - || MessagingTime.IsExpired(now, previous.EnqueuedAt!.Value, Options.MaxOutboxRetryAge)) + if (outcome.Expired + || status is not (null or DeliveryStatus.Backpressured or DeliveryStatus.HandlerNotFound) + || status is DeliveryStatus.HandlerNotFound && MessagingTime.IsExpired(now, handlerNotFoundSince!.Value, Options.MaxHandlerNotFoundRetryAge) + || unansweredSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge)) { var letter = new OutboxDeadLetter { @@ -583,12 +590,15 @@ private void ApplyDeadLetters(Dictionary entries) AddDeadLetter(ref deadLetters, letter); return (null, letter); } - var exponent = answered ? Math.Min(attemptCount - 1, DurableMessagingOptions.MaximumBackoffExponent) : DurableMessagingOptions.MaximumBackoffExponent; + + // Failed attempts return to the outbox with a durable backoff, so later batches deliver other messages meanwhile. + var exponent = Math.Min(attemptCount - 1, DurableMessagingOptions.MaximumBackoffExponent); return (new OutboxMessageState { AttemptCount = attemptCount, LastError = error, - EnqueuedAt = previous.EnqueuedAt, + UnansweredSince = unansweredSince, + HandlerNotFoundSince = handlerNotFoundSince, NextAttemptAt = MessagingTime.AddClamped(now, TimeSpan.FromTicks(Options.BackpressureRetryDelay.Ticks * (1L << exponent))) }, null); } @@ -623,7 +633,7 @@ private void RecordDeliveryMetrics(DeliveryOutcome outcome) LogDeliveryBackpressured(Logger, message.Envelope.MessageId, message.ReceiverId); break; case DeliveryStatus.HandlerNotFound: - if (Logger.IsEnabled(LogLevel.Warning)) + if (Logger.IsEnabled(LogLevel.Error)) { LogDeliveryHandlerNotFound(Logger, message.Envelope.MessageId, message.ReceiverId, outcome.Result?.Message); } @@ -637,14 +647,16 @@ private void RecordDeliveryMetrics(DeliveryOutcome outcome) } } + private bool IsRetryExpired(OutboxMessageState state, DateTimeOffset now) => + state.UnansweredSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge); + private bool IsReadyForAttempt(OutboxMessageState state, DateTimeOffset now) => - state.EnqueuedAt is { } enqueuedAt && MessagingTime.IsExpired(now, enqueuedAt, Options.MaxOutboxRetryAge) - || state.NextAttemptAt is null || state.NextAttemptAt <= now; + IsRetryExpired(state, now) || state.NextAttemptAt is null || state.NextAttemptAt <= now; private DateTimeOffset GetNextAttemptAt(OutboxMessageState state, DateTimeOffset now) { var retryAt = state.NextAttemptAt ?? now; - var expiresAt = state.EnqueuedAt is { } enqueuedAt ? MessagingTime.AddClamped(enqueuedAt, Options.MaxOutboxRetryAge) : retryAt; + var expiresAt = state.UnansweredSince is { } since ? MessagingTime.AddClamped(since, Options.MaxOutboxRetryAge) : retryAt; return retryAt <= expiresAt ? retryAt : expiresAt; } @@ -704,7 +716,7 @@ internal async Task StartAsync(CancellationToken cancellationToken) [LoggerMessage(Level = LogLevel.Debug, Message = "Delivered message {MessageId} to {ReceiverId} (Status: {Status})")] private static partial void LogMessageDelivered(ILogger logger, MessageId messageId, GrainId receiverId, DeliveryStatus status); - [LoggerMessage(Level = LogLevel.Warning, Message = "No inbox handler for message {MessageId} to {ReceiverId}: {Message}")] + [LoggerMessage(Level = LogLevel.Error, Message = "No inbox handler for message {MessageId} to {ReceiverId}: {Message}")] private static partial void LogDeliveryHandlerNotFound(ILogger logger, MessageId messageId, GrainId receiverId, string? message); [LoggerMessage(Level = LogLevel.Debug, Message = "Backpressured delivering message {MessageId} to {ReceiverId}, will retry later")] @@ -728,5 +740,12 @@ internal async Task StartAsync(CancellationToken cancellationToken) [LoggerMessage(Level = LogLevel.Information, Message = "Outbox delivery complete: {DeliveredCount} delivered, {BackpressuredCount} backpressured, {FailedCount} failed, {RemainingCount} remaining")] private static partial void LogDeliveryComplete(ILogger logger, int deliveredCount, int backpressuredCount, int failedCount, int remainingCount); - private readonly record struct DeliveryOutcome(OutboxEntry Entry, OutboxMessage Message, DeliveryResult? Result, string? Error, TimeSpan Duration, bool Expired); + private readonly record struct DeliveryOutcome( + OutboxEntry Entry, + OutboxMessage Message, + DeliveryResult? Result, + string? Error, + TimeSpan Duration, + DateTimeOffset StartedAt, + bool Expired); } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 07d289a29cb..95c6b08cdb2 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -174,14 +174,14 @@ write completes. Each receiver returns a `DeliveryResult`: - `DeadLettered`: the receiver moved the message to its dead-letter store. `Accepted`, `Duplicate`, and `DeadLettered` are terminal: the outbox removes the -message. Each delivery attempt runs through the `DeliveryResiliencePipeline` Polly -pipeline. By default, it retries failed calls five times with exponential backoff -within a total timeout of six response timeouts, and a silo-wide circuit breaker -opens on correlated failures. The outbox retries `Backpressured`, `HandlerNotFound`, -and calls which fail through the pipeline with exponential backoff. It dead-letters -a message after `MaxDeliveryAttempts` replies without acceptance, or once it is -older than `MaxOutboxRetryAge`, so failed calls during an outage retry until that -age. Delivery order is unspecified. +message. Any other outcome schedules a durable retry with exponential backoff, +and later batches deliver other messages meanwhile. The outbox retries +`Backpressured` until the receiver takes the message. It logs `HandlerNotFound` as +an error and dead-letters the message after `MaxHandlerNotFoundRetryAge`. Calls +which fail without an answer, such as during an outage, retry for +`MaxOutboxRetryAge` from the first unanswered attempt. Each attempt runs through the +`DeliveryResiliencePipeline` Polly pipeline, which can add timeouts, retries, or a +circuit breaker. Delivery order is unspecified. Each activation runs one messaging pump on its scheduler, processing inbox batches and delivering outbox batches. One durable job per grain wakes the pump while either diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 646951c4b8a..3d728277a8a 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -72,7 +72,7 @@ public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWh Assert.Null(JsonSerializer.Deserialize(immediateJson, InboxJsonContext.Default.InboxCommand)!.Message!.Value.Envelope.DeliverAt); Assert.Null(serializer.Deserialize(serializer.SerializeToArray(immediate))!.Message!.Value.Envelope.DeliverAt); Assert.DoesNotContain("DeliverAt", - JsonSerializer.Serialize(new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("immediate"), State = new() { EnqueuedAt = Now } }, + JsonSerializer.Serialize(new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("immediate"), State = new() }, OutboxJsonContext.Default.OutboxCommand), StringComparison.Ordinal); } @@ -80,7 +80,7 @@ public void DeliverAt_RoundTripsThroughJsonAndOrleansSerializationAndIsOmittedWh [Fact] public void ReferenceEnqueue_CarriesNoMessageInJsonAndOrleansSerialization() { - var command = new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() { EnqueuedAt = Now } }; + var command = new OutboxCommand { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() }; using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); var serializer = services.GetRequiredService>(); @@ -145,10 +145,10 @@ private static OutboxCommand[] CreateOutboxCommands() var deadLetter = new OutboxDeadLetter { Message = OutboxMessage("dead"), DeadLetteredAt = Now, Reason = "rejected", AttemptCount = 3 }; return [ - new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() { EnqueuedAt = Now } }, - new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() { EnqueuedAt = Now } }, - new() { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() { EnqueuedAt = Now } }, - new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", EnqueuedAt = Now } }, + new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() }, + new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() }, + new() { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", UnansweredSince = Now, HandlerNotFoundSince = Now } }, new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "dead"), DeadLetter = deadLetter }, new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [MessageId.Create("orders", "dead")] }, new() @@ -158,8 +158,8 @@ private static OutboxCommand[] CreateOutboxCommands() { Messages = [ - new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, EnqueuedAt = Now } }, - new() { Reference = Reference("pending-reference"), State = new() { AttemptCount = 2, EnqueuedAt = Now } } + new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, UnansweredSince = Now } }, + new() { Reference = Reference("pending-reference"), State = new() { AttemptCount = 2, HandlerNotFoundSince = Now } } ], DeadLetters = [deadLetter] } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index ec3cddf0485..5b4271b98f1 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -104,7 +104,7 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); - // The receiver has no handler, so both permitted delivery attempts fail. + // The receiver has no handler, so delivery attempts fail until MaxHandlerNotFoundRetryAge elapses. await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); var (state, deadLetters) = await OnGrainAsync(sender, context => ( Grain(context).GetState(), @@ -116,8 +116,8 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe Assert.Equal(receiver.GetGrainId(), letter.Message.ReceiverId); Assert.Equal(payload, letter.Message.Envelope.Payload); Assert.Equal(SourceSinkGrain.Subject, letter.Message.Envelope.Subject); - Assert.Equal(2, letter.AttemptCount); - Assert.Equal(3, Probe.For(sender).Materializations); + Assert.True(letter.AttemptCount > 1); + Assert.Equal(letter.AttemptCount + 1, Probe.For(sender).Materializations); // The dead letter keeps its own copy for diagnosis. await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 2); @@ -412,7 +412,7 @@ private static void ApplyReference(Outbox outbox, MessageId messageId) => { Kind = OutboxCommandKind.Enqueue, Reference = new() { MessageId = messageId }, - State = new() { EnqueuedAt = DateTimeOffset.UtcNow } + State = new() }); private static void AssertSourceFailureDeadLetter(DeadLetter letter, MessageId messageId, SourceFault fault) diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs index 827255bdddd..262f07b8115 100644 --- a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs @@ -1,10 +1,9 @@ -using Microsoft.Extensions.Options; -using Orleans.Configuration; +using System.Collections.Concurrent; +using System.Diagnostics; using Orleans.Messaging.Configuration; using Orleans.Messaging.Tests.Support; using Orleans.Runtime; using Polly; -using Polly.CircuitBreaker; using Polly.Retry; using Xunit; @@ -27,9 +26,9 @@ public async Task FailedTry_IsRetriedWithinTheDeliveryAttempt() MaxRetryAttempts = InPipelineRetries, Delay = TimeSpan.Zero, }) - .AddStrategy(_ => strategy, new FaultStrategyOptions()) + .AddStrategy(_ => strategy, new ScriptedStrategyOptions()) .Build()); - fixture.Strategy.FailNext(InPipelineRetries); + fixture.Strategy.Script(Step.Unanswered(), Step.Unanswered()); var (sender, receiver) = await SendAsync(fixture, "retried-in-attempt"); @@ -41,13 +40,11 @@ public async Task FailedTry_IsRetriedWithinTheDeliveryAttempt() } [Fact] - public async Task FailuresLeavingThePipeline_RetryDurablyWithoutConsumingDeliveryAttempts() + public async Task FailuresLeavingThePipeline_RetryInLaterBatches() { - await using var fixture = await StartAsync(static strategy => new ResiliencePipelineBuilder() - .AddStrategy(_ => strategy, new FaultStrategyOptions()) - .Build()); + await using var fixture = await StartAsync(Scripted); const int FailedAttempts = 3; - fixture.Strategy.FailNext(FailedAttempts); + fixture.Strategy.Script(Step.Unanswered(), Step.Unanswered(), Step.Unanswered()); var (sender, receiver) = await SendAsync(fixture, "retried-durably"); @@ -72,35 +69,105 @@ public async Task EmptyPipeline_DeliversDirectly() } [Fact] - public async Task DefaultPipeline_RetriesFailedCallsAndPropagatesOpenCircuit() + public async Task Backpressured_RetriesBeyondEveryRetryAgeUntilTheReceiverTakesTheMessage() { - Assert.Null(new DurableMessagingOptions().DeliveryResiliencePipeline); - var pipeline = new DefaultDeliveryResiliencePipeline( - Options.Create(new SiloMessagingOptions { ResponseTimeout = TimeSpan.FromMilliseconds(100) }), - TimeProvider.System).Pipeline; - var cancellationToken = TestContext.Current.CancellationToken; - - var calls = 0; - var result = await pipeline.ExecuteAsync( - _ => ++calls == 1 ? throw new IOException("Injected outage.") : ValueTask.FromResult(DeliveryResult.Accepted()), - cancellationToken); - Assert.Equal(DeliveryStatus.Accepted, result.Status); - Assert.Equal(2, calls); - - calls = 0; - await Assert.ThrowsAsync(() => pipeline.ExecuteAsync( - _ => - { - calls++; - throw new BrokenCircuitException(); - }, - cancellationToken).AsTask()); - Assert.Equal(1, calls); + var retryAge = TimeSpan.FromMilliseconds(100); + await using var fixture = await StartAsync(Scripted, options => + { + options.MaxOutboxRetryAge = retryAge; + options.MaxHandlerNotFoundRetryAge = retryAge; + }); + const int BackpressuredAttempts = 8; + fixture.Strategy.Script(Enumerable.Repeat(Step.Answer(DeliveryResult.Backpressured()), BackpressuredAttempts).ToArray()); + var elapsed = Stopwatch.StartNew(); + + var (sender, receiver) = await SendAsync(fixture, "backpressured"); + + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.True(elapsed.Elapsed > retryAge); + Assert.Equal("backpressured", Assert.Single(delivered.Effects).Value); + Assert.Equal(BackpressuredAttempts + 1, fixture.Strategy.Executions); + Assert.Empty((await fixture.WaitForOutboxCountAsync(sender, 0)).OutboxDeadLetters); + } + + [Fact] + public async Task UnansweredAttempts_DeadLetterAfterMaxOutboxRetryAge() + { + await using var fixture = await StartAsync(Scripted, static options => options.MaxOutboxRetryAge = TimeSpan.FromMilliseconds(100)); + fixture.Strategy.Repeat(Step.Unanswered()); + + var (sender, receiver) = await SendAsync(fixture, "unanswered"); + + var deadLetter = Assert.Single((await fixture.WaitForDeadLetterCountAsync(sender, 1)).OutboxDeadLetters); + Assert.True(deadLetter.AttemptCount > 1); + Assert.Equal(fixture.Strategy.Executions, deadLetter.AttemptCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task AReceiverAnswer_RestartsTheUnansweredRetryAge() + { + // Each unanswered span stays within the retry age, while both spans together exceed it. + var timeout = TimeSpan.FromMilliseconds(1600); + await using var fixture = await StartAsync(Scripted, static options => + { + options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3); + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(1); + }); + fixture.Strategy.Script(Step.Unanswered(timeout), Step.Answer(DeliveryResult.Backpressured()), Step.Unanswered(timeout)); + + var (sender, receiver) = await SendAsync(fixture, "answered-between-outages"); + + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("answered-between-outages", Assert.Single(delivered.Effects).Value); + Assert.Equal(4, fixture.Strategy.Executions); + Assert.Empty((await fixture.WaitForOutboxCountAsync(sender, 0)).OutboxDeadLetters); + } + + [Fact] + public async Task HandlerNotFound_DeadLettersAfterMaxHandlerNotFoundRetryAge() + { + await using var fixture = await StartAsync(Scripted, static options => options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(100)); + fixture.Strategy.Repeat(Step.Answer(DeliveryResult.HandlerNotFound())); + + var (sender, receiver) = await SendAsync(fixture, "unhandled"); + + var deadLetter = Assert.Single((await fixture.WaitForDeadLetterCountAsync(sender, 1)).OutboxDeadLetters); + Assert.True(deadLetter.AttemptCount > 1); + Assert.Equal(DeliveryResult.HandlerNotFound().Message, deadLetter.Reason); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + + [Fact] + public async Task AnotherReceiverAnswer_RestartsTheHandlerNotFoundRetryAge() + { + await using var fixture = await StartAsync(Scripted, static options => + { + options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(400); + options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(1); + }); + fixture.Strategy.Script( + Step.Answer(DeliveryResult.HandlerNotFound()), + Step.Answer(DeliveryResult.Backpressured(), TimeSpan.FromMilliseconds(600)), + Step.Answer(DeliveryResult.HandlerNotFound())); + + var (sender, receiver) = await SendAsync(fixture, "handler-registered"); + + var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); + Assert.Equal("handler-registered", Assert.Single(delivered.Effects).Value); + Assert.Equal(4, fixture.Strategy.Executions); + Assert.Empty((await fixture.WaitForOutboxCountAsync(sender, 0)).OutboxDeadLetters); } - private static async Task StartAsync(Func createPipeline) + private static ResiliencePipeline Scripted(ScriptedStrategy strategy) => new ResiliencePipelineBuilder() + .AddStrategy(_ => strategy, new ScriptedStrategyOptions()) + .Build(); + + private static async Task StartAsync( + Func createPipeline, + Action? configure = null) { - var fixture = new DeliveryResilienceClusterFixture(createPipeline); + var fixture = new DeliveryResilienceClusterFixture(createPipeline, configure); await fixture.InitializeAsync(); return fixture; } @@ -116,29 +183,45 @@ await sender.SendAsync( return (sender, receiver); } - private sealed class DeliveryResilienceClusterFixture(Func createPipeline) + private sealed class DeliveryResilienceClusterFixture( + Func createPipeline, + Action? configure) : MessagingClusterFixture(initialSilos: 1) { - public FaultStrategy Strategy { get; } = new(); + public ScriptedStrategy Strategy { get; } = new(); protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); - options.MaxDeliveryAttempts = 1; options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(5); options.DeliveryResiliencePipeline = createPipeline(Strategy); + configure?.Invoke(options); } } - /// Fails a configured number of tries before calling the receiver. - private sealed class FaultStrategy : ResilienceStrategy + /// A scripted outcome of one try: a receiver answer, or a failure without an answer. + private readonly record struct Step(DeliveryResult? Result, TimeSpan Delay) { - private int _failures; + public static Step Answer(DeliveryResult result, TimeSpan delay = default) => new(result, delay); + + public static Step Unanswered(TimeSpan delay = default) => new(null, delay); + } + + /// Plays scripted steps in place of calling the receiver, then repeats a step or calls the receiver. + private sealed class ScriptedStrategy : ResilienceStrategy + { + private readonly ConcurrentQueue _steps = new(); + private Step? _then; private int _executions; public int Executions => Volatile.Read(ref _executions); - public void FailNext(int count) => Volatile.Write(ref _failures, count); + public void Script(params Step[] steps) + { + foreach (var step in steps) _steps.Enqueue(step); + } + + public void Repeat(Step step) => _then = step; protected override ValueTask> ExecuteCore( Func>> callback, @@ -146,11 +229,31 @@ protected override ValueTask> ExecuteCore( TState state) { Interlocked.Increment(ref _executions); - return Interlocked.Decrement(ref _failures) >= 0 - ? ValueTask.FromResult(Outcome.FromException(new TimeoutException("Injected delivery failure."))) - : callback(context, state); + if (!_steps.TryDequeue(out var step)) + { + if (_then is not { } then) + { + return callback(context, state); + } + + step = then; + } + + return PlayAsync(step, context.CancellationToken); + } + + private static async ValueTask> PlayAsync(Step step, CancellationToken cancellationToken) + { + if (step.Delay > TimeSpan.Zero) + { + await Task.Delay(step.Delay, cancellationToken); + } + + return step.Result is { } result + ? Outcome.FromResult((TResult)(object)result) + : Outcome.FromException(new TimeoutException("Injected delivery failure.")); } } - private sealed class FaultStrategyOptions : ResilienceStrategyOptions; + private sealed class ScriptedStrategyOptions : ResilienceStrategyOptions; } diff --git a/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs b/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs index 7fe9e198021..b2c01aded23 100644 --- a/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/PublicMessagingRoutingAndDeadLetterTests.cs @@ -35,7 +35,7 @@ await sender.SendAsync( Assert.Equal(0, senderState.OutboxCount); var deadLetter = Assert.Single(senderState.OutboxDeadLetters); Assert.Equal("unknown/outbox-route", deadLetter.Route); - Assert.Equal(3, deadLetter.AttemptCount); + Assert.True(deadLetter.AttemptCount > 1); Assert.Contains("No inbox handler", deadLetter.Reason, StringComparison.Ordinal); var receiverState = await receiver.GetSnapshotAsync(); Assert.Equal(0, receiverState.InboxCount); diff --git a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs index 1969f147cef..98e1671b0e5 100644 --- a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs @@ -301,7 +301,7 @@ public async Task RecoveredOutboxEntryAddressedToItsOwnGrain_IsRejected() { Kind = OutboxCommandKind.Enqueue, Message = message, - State = new() { EnqueuedAt = Fixture.Clock.GetUtcNow() } + State = new() }); ((IStateMachine)outbox).OnRecoveryCompleted(); })); From ed0f9ebe1718fd50bff5f3c2975d8346837c90fb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 13 Oct 2026 19:20:03 -0700 Subject: [PATCH 77/95] refactor(messaging)!: track one outbox retry period and drop MaxHandlerNotFoundRetryAge OutboxMessageState records a single RetryingSince, which starts with the first failed attempt and restarts on each Backpressured answer. MaxOutboxRetryAge (now 6 hours) bounds both unanswered attempts and HandlerNotFound answers, and dead letters keep the last failure as their reason. --- src/Orleans.Messaging/MessageState.cs | 8 +--- src/Orleans.Messaging/Outbox.cs | 24 ++++-------- src/Orleans.Messaging/README.md | 6 +-- .../MessagingCommandSerializationTests.cs | 6 +-- .../Functional/MessageSourceAndSinkTests.cs | 5 ++- .../OutboxDeliveryResilienceTests.cs | 39 ++++--------------- 6 files changed, 26 insertions(+), 62 deletions(-) diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs index 7c75bba4986..fe69de221fd 100644 --- a/src/Orleans.Messaging/MessageState.cs +++ b/src/Orleans.Messaging/MessageState.cs @@ -35,13 +35,9 @@ internal sealed class OutboxMessageState [Id(2)] public string? LastError { get; set; } - /// Gets or sets when the first delivery attempt without a receiver answer started, since the receiver last answered. + /// Gets or sets when the current run of failed attempts started; a backpressured attempt ends the run. [Id(3)] - public DateTimeOffset? UnansweredSince { get; set; } - - /// Gets or sets when the receiver first reported no inbox handler, since it last answered otherwise. - [Id(4)] - public DateTimeOffset? HandlerNotFoundSince { get; set; } + public DateTimeOffset? RetryingSince { get; set; } } [GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 37a553007ad..4a1ec925dca 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -327,7 +327,7 @@ private async Task DeliverAsync(OutboxEntry entry, OutboxMessag var startedAt = TimeProvider.GetUtcNow(); if (IsRetryExpired(entry.State, startedAt)) { - return new(entry, message, null, $"The receiver did not answer delivery attempts within {Options.MaxOutboxRetryAge}.", TimeSpan.Zero, startedAt, true); + return new(entry, message, null, entry.State.LastError, TimeSpan.Zero, startedAt, true); } var start = Stopwatch.GetTimestamp(); try @@ -559,16 +559,10 @@ private void ApplyDeadLetters(Dictionary entries) var now = TimeProvider.GetUtcNow(); var attemptCount = outcome.Expired ? previous.AttemptCount : checked(previous.AttemptCount + 1); - // A receiver checks its deduplication records before it answers, so an answer proves that no earlier attempt was - // accepted. Only attempts since the receiver last answered can have been accepted, which bounds their retry by + // A receiver checks its deduplication records before it answers, so a Backpressured answer proves that no earlier + // attempt was accepted. Only attempts since then can have been accepted, which bounds their retry by // MaxOutboxRetryAge, within the receiver's deduplication window. - var unansweredSince = outcome.Result is null ? previous.UnansweredSince ?? outcome.StartedAt : (DateTimeOffset?)null; - var handlerNotFoundSince = status switch - { - null => previous.HandlerNotFoundSince, - DeliveryStatus.HandlerNotFound => previous.HandlerNotFoundSince ?? outcome.StartedAt, - _ => null - }; + var retryingSince = status is DeliveryStatus.Backpressured ? (DateTimeOffset?)null : previous.RetryingSince ?? outcome.StartedAt; var error = outcome.Error ?? status switch { DeliveryStatus.Backpressured => "The receiver is backpressured.", @@ -577,8 +571,7 @@ private void ApplyDeadLetters(Dictionary entries) }; if (outcome.Expired || status is not (null or DeliveryStatus.Backpressured or DeliveryStatus.HandlerNotFound) - || status is DeliveryStatus.HandlerNotFound && MessagingTime.IsExpired(now, handlerNotFoundSince!.Value, Options.MaxHandlerNotFoundRetryAge) - || unansweredSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge)) + || retryingSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge)) { var letter = new OutboxDeadLetter { @@ -597,8 +590,7 @@ private void ApplyDeadLetters(Dictionary entries) { AttemptCount = attemptCount, LastError = error, - UnansweredSince = unansweredSince, - HandlerNotFoundSince = handlerNotFoundSince, + RetryingSince = retryingSince, NextAttemptAt = MessagingTime.AddClamped(now, TimeSpan.FromTicks(Options.BackpressureRetryDelay.Ticks * (1L << exponent))) }, null); } @@ -648,7 +640,7 @@ private void RecordDeliveryMetrics(DeliveryOutcome outcome) } private bool IsRetryExpired(OutboxMessageState state, DateTimeOffset now) => - state.UnansweredSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge); + state.RetryingSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge); private bool IsReadyForAttempt(OutboxMessageState state, DateTimeOffset now) => IsRetryExpired(state, now) || state.NextAttemptAt is null || state.NextAttemptAt <= now; @@ -656,7 +648,7 @@ private bool IsReadyForAttempt(OutboxMessageState state, DateTimeOffset now) => private DateTimeOffset GetNextAttemptAt(OutboxMessageState state, DateTimeOffset now) { var retryAt = state.NextAttemptAt ?? now; - var expiresAt = state.UnansweredSince is { } since ? MessagingTime.AddClamped(since, Options.MaxOutboxRetryAge) : retryAt; + var expiresAt = state.RetryingSince is { } since ? MessagingTime.AddClamped(since, Options.MaxOutboxRetryAge) : retryAt; return retryAt <= expiresAt ? retryAt : expiresAt; } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 95c6b08cdb2..95b3ba5a808 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -177,9 +177,9 @@ write completes. Each receiver returns a `DeliveryResult`: message. Any other outcome schedules a durable retry with exponential backoff, and later batches deliver other messages meanwhile. The outbox retries `Backpressured` until the receiver takes the message. It logs `HandlerNotFound` as -an error and dead-letters the message after `MaxHandlerNotFoundRetryAge`. Calls -which fail without an answer, such as during an outage, retry for -`MaxOutboxRetryAge` from the first unanswered attempt. Each attempt runs through the +an error. Other failures, including calls which fail without an answer during an +outage, dead-letter the message after `MaxOutboxRetryAge` (6 hours) from the first +failed attempt since the last `Backpressured` answer. Each attempt runs through the `DeliveryResiliencePipeline` Polly pipeline, which can add timeouts, retries, or a circuit breaker. Delivery order is unspecified. diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 3d728277a8a..7afef2d99e6 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -148,7 +148,7 @@ private static OutboxCommand[] CreateOutboxCommands() new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() }, new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() }, new() { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() }, - new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", UnansweredSince = Now, HandlerNotFoundSince = Now } }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", RetryingSince = Now } }, new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "dead"), DeadLetter = deadLetter }, new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [MessageId.Create("orders", "dead")] }, new() @@ -158,8 +158,8 @@ private static OutboxCommand[] CreateOutboxCommands() { Messages = [ - new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, UnansweredSince = Now } }, - new() { Reference = Reference("pending-reference"), State = new() { AttemptCount = 2, HandlerNotFoundSince = Now } } + new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, RetryingSince = Now } }, + new() { Reference = Reference("pending-reference"), State = new() { AttemptCount = 2 } } ], DeadLetters = [deadLetter] } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 5b4271b98f1..d6bac244bfc 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -104,7 +104,7 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); - // The receiver has no handler, so delivery attempts fail until MaxHandlerNotFoundRetryAge elapses. + // The receiver has no handler, so delivery attempts fail until MaxOutboxRetryAge elapses. await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); var (state, deadLetters) = await OnGrainAsync(sender, context => ( Grain(context).GetState(), @@ -117,7 +117,8 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe Assert.Equal(payload, letter.Message.Envelope.Payload); Assert.Equal(SourceSinkGrain.Subject, letter.Message.Envelope.Subject); Assert.True(letter.AttemptCount > 1); - Assert.Equal(letter.AttemptCount + 1, Probe.For(sender).Materializations); + // Each attempt materializes the message, as does a final wake which finds the retry period expired. + Assert.InRange(Probe.For(sender).Materializations, letter.AttemptCount + 1, letter.AttemptCount + 2); // The dead letter keeps its own copy for diagnosis. await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 2); diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs index 262f07b8115..44848e2b60f 100644 --- a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs @@ -69,14 +69,10 @@ public async Task EmptyPipeline_DeliversDirectly() } [Fact] - public async Task Backpressured_RetriesBeyondEveryRetryAgeUntilTheReceiverTakesTheMessage() + public async Task Backpressured_RetriesBeyondTheRetryAgeUntilTheReceiverTakesTheMessage() { var retryAge = TimeSpan.FromMilliseconds(100); - await using var fixture = await StartAsync(Scripted, options => - { - options.MaxOutboxRetryAge = retryAge; - options.MaxHandlerNotFoundRetryAge = retryAge; - }); + await using var fixture = await StartAsync(Scripted, options => options.MaxOutboxRetryAge = retryAge); const int BackpressuredAttempts = 8; fixture.Strategy.Script(Enumerable.Repeat(Step.Answer(DeliveryResult.Backpressured()), BackpressuredAttempts).ToArray()); var elapsed = Stopwatch.StartNew(); @@ -105,16 +101,16 @@ public async Task UnansweredAttempts_DeadLetterAfterMaxOutboxRetryAge() } [Fact] - public async Task AReceiverAnswer_RestartsTheUnansweredRetryAge() + public async Task Backpressured_RestartsTheRetryAge() { - // Each unanswered span stays within the retry age, while both spans together exceed it. + // Each failing span stays within the retry age, while both spans together exceed it. var timeout = TimeSpan.FromMilliseconds(1600); await using var fixture = await StartAsync(Scripted, static options => { options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3); options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(1); }); - fixture.Strategy.Script(Step.Unanswered(timeout), Step.Answer(DeliveryResult.Backpressured()), Step.Unanswered(timeout)); + fixture.Strategy.Script(Step.Unanswered(timeout), Step.Answer(DeliveryResult.Backpressured()), Step.Answer(DeliveryResult.HandlerNotFound(), timeout)); var (sender, receiver) = await SendAsync(fixture, "answered-between-outages"); @@ -125,9 +121,9 @@ public async Task AReceiverAnswer_RestartsTheUnansweredRetryAge() } [Fact] - public async Task HandlerNotFound_DeadLettersAfterMaxHandlerNotFoundRetryAge() + public async Task HandlerNotFound_DeadLettersAfterMaxOutboxRetryAge() { - await using var fixture = await StartAsync(Scripted, static options => options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(100)); + await using var fixture = await StartAsync(Scripted, static options => options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3)); fixture.Strategy.Repeat(Step.Answer(DeliveryResult.HandlerNotFound())); var (sender, receiver) = await SendAsync(fixture, "unhandled"); @@ -138,27 +134,6 @@ public async Task HandlerNotFound_DeadLettersAfterMaxHandlerNotFoundRetryAge() Assert.Empty((await receiver.GetSnapshotAsync()).Effects); } - [Fact] - public async Task AnotherReceiverAnswer_RestartsTheHandlerNotFoundRetryAge() - { - await using var fixture = await StartAsync(Scripted, static options => - { - options.MaxHandlerNotFoundRetryAge = TimeSpan.FromMilliseconds(400); - options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(1); - }); - fixture.Strategy.Script( - Step.Answer(DeliveryResult.HandlerNotFound()), - Step.Answer(DeliveryResult.Backpressured(), TimeSpan.FromMilliseconds(600)), - Step.Answer(DeliveryResult.HandlerNotFound())); - - var (sender, receiver) = await SendAsync(fixture, "handler-registered"); - - var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); - Assert.Equal("handler-registered", Assert.Single(delivered.Effects).Value); - Assert.Equal(4, fixture.Strategy.Executions); - Assert.Empty((await fixture.WaitForOutboxCountAsync(sender, 0)).OutboxDeadLetters); - } - private static ResiliencePipeline Scripted(ScriptedStrategy strategy) => new ResiliencePipelineBuilder() .AddStrategy(_ => strategy, new ScriptedStrategyOptions()) .Build(); From 9efef21ce75019971db4c00cae8921157e7ca40b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 08:43:03 -0700 Subject: [PATCH 78/95] refactor(messaging)!: align outbox retry terminology with Temporal Rename the retry options and state after Temporal's retry policy: - DurableMessagingOptions.BackpressureRetryDelay becomes RetryInitialInterval. - DurableMessagingOptions.MaxOutboxRetryAge becomes OutboxRetryTimeout. - OutboxMessageState.RetryingSince becomes ExpirationTime and stores the deadline: the first failed attempt plus OutboxRetryTimeout. A Backpressured answer clears it, and the next failed attempt sets a new deadline. BREAKING CHANGE: BackpressureRetryDelay and MaxOutboxRetryAge are renamed to RetryInitialInterval and OutboxRetryTimeout. --- src/Orleans.Messaging/MessageState.cs | 4 +-- src/Orleans.Messaging/Outbox.cs | 20 +++++++------- src/Orleans.Messaging/README.md | 2 +- .../MessagingCommandSerializationTests.cs | 4 +-- .../Functional/MessageSourceAndSinkTests.cs | 4 +-- .../OutboxDeliveryResilienceTests.cs | 26 +++++++++---------- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs index fe69de221fd..3a055ce7e01 100644 --- a/src/Orleans.Messaging/MessageState.cs +++ b/src/Orleans.Messaging/MessageState.cs @@ -35,9 +35,9 @@ internal sealed class OutboxMessageState [Id(2)] public string? LastError { get; set; } - /// Gets or sets when the current run of failed attempts started; a backpressured attempt ends the run. + /// Gets or sets when the outbox stops retrying and dead-letters the message; a backpressured attempt clears it. [Id(3)] - public DateTimeOffset? RetryingSince { get; set; } + public DateTimeOffset? ExpirationTime { get; set; } } [GenerateSerializer, Alias("Orleans.Messaging.InboxDeadLetter")] diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 4a1ec925dca..005034ce906 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -560,9 +560,11 @@ private void ApplyDeadLetters(Dictionary entries) var attemptCount = outcome.Expired ? previous.AttemptCount : checked(previous.AttemptCount + 1); // A receiver checks its deduplication records before it answers, so a Backpressured answer proves that no earlier - // attempt was accepted. Only attempts since then can have been accepted, which bounds their retry by - // MaxOutboxRetryAge, within the receiver's deduplication window. - var retryingSince = status is DeliveryStatus.Backpressured ? (DateTimeOffset?)null : previous.RetryingSince ?? outcome.StartedAt; + // attempt was accepted. Only attempts since then can have been accepted, so retries expire OutboxRetryTimeout after + // the first of them, within the receiver's deduplication window. + var expirationTime = status is DeliveryStatus.Backpressured + ? (DateTimeOffset?)null + : previous.ExpirationTime ?? MessagingTime.AddClamped(outcome.StartedAt, Options.OutboxRetryTimeout); var error = outcome.Error ?? status switch { DeliveryStatus.Backpressured => "The receiver is backpressured.", @@ -571,7 +573,7 @@ private void ApplyDeadLetters(Dictionary entries) }; if (outcome.Expired || status is not (null or DeliveryStatus.Backpressured or DeliveryStatus.HandlerNotFound) - || retryingSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge)) + || expirationTime <= now) { var letter = new OutboxDeadLetter { @@ -590,8 +592,8 @@ private void ApplyDeadLetters(Dictionary entries) { AttemptCount = attemptCount, LastError = error, - RetryingSince = retryingSince, - NextAttemptAt = MessagingTime.AddClamped(now, TimeSpan.FromTicks(Options.BackpressureRetryDelay.Ticks * (1L << exponent))) + ExpirationTime = expirationTime, + NextAttemptAt = MessagingTime.AddClamped(now, TimeSpan.FromTicks(Options.RetryInitialInterval.Ticks * (1L << exponent))) }, null); } @@ -639,8 +641,7 @@ private void RecordDeliveryMetrics(DeliveryOutcome outcome) } } - private bool IsRetryExpired(OutboxMessageState state, DateTimeOffset now) => - state.RetryingSince is { } since && MessagingTime.IsExpired(now, since, Options.MaxOutboxRetryAge); + private static bool IsRetryExpired(OutboxMessageState state, DateTimeOffset now) => state.ExpirationTime <= now; private bool IsReadyForAttempt(OutboxMessageState state, DateTimeOffset now) => IsRetryExpired(state, now) || state.NextAttemptAt is null || state.NextAttemptAt <= now; @@ -648,8 +649,7 @@ private bool IsReadyForAttempt(OutboxMessageState state, DateTimeOffset now) => private DateTimeOffset GetNextAttemptAt(OutboxMessageState state, DateTimeOffset now) { var retryAt = state.NextAttemptAt ?? now; - var expiresAt = state.RetryingSince is { } since ? MessagingTime.AddClamped(since, Options.MaxOutboxRetryAge) : retryAt; - return retryAt <= expiresAt ? retryAt : expiresAt; + return state.ExpirationTime < retryAt ? state.ExpirationTime.Value : retryAt; } /// diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 95b3ba5a808..7b5e42ad1c4 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -178,7 +178,7 @@ message. Any other outcome schedules a durable retry with exponential backoff, and later batches deliver other messages meanwhile. The outbox retries `Backpressured` until the receiver takes the message. It logs `HandlerNotFound` as an error. Other failures, including calls which fail without an answer during an -outage, dead-letter the message after `MaxOutboxRetryAge` (6 hours) from the first +outage, dead-letter the message after `OutboxRetryTimeout` (6 hours) from the first failed attempt since the last `Backpressured` answer. Each attempt runs through the `DeliveryResiliencePipeline` Polly pipeline, which can add timeouts, retries, or a circuit breaker. Delivery order is unspecified. diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 7afef2d99e6..5bcf37eb89b 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -148,7 +148,7 @@ private static OutboxCommand[] CreateOutboxCommands() new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("enqueue"), State = new() }, new() { Kind = OutboxCommandKind.Enqueue, Message = OutboxMessage("scheduled", Now), State = new() }, new() { Kind = OutboxCommandKind.Enqueue, Reference = Reference("reference"), State = new() }, - new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", RetryingSince = Now } }, + new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "retry"), State = new() { AttemptCount = 2, NextAttemptAt = Now, LastError = "boom", ExpirationTime = Now } }, new() { Kind = OutboxCommandKind.Delivery, MessageId = MessageId.Create("orders", "dead"), DeadLetter = deadLetter }, new() { Kind = OutboxCommandKind.RemoveDeadLetters, RemovedDeadLetters = [MessageId.Create("orders", "dead")] }, new() @@ -158,7 +158,7 @@ private static OutboxCommand[] CreateOutboxCommands() { Messages = [ - new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, RetryingSince = Now } }, + new() { Message = OutboxMessage("pending", Now), State = new() { AttemptCount = 1, ExpirationTime = Now } }, new() { Reference = Reference("pending-reference"), State = new() { AttemptCount = 2 } } ], DeadLetters = [deadLetter] diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index d6bac244bfc..3653911a06e 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -104,7 +104,7 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); - // The receiver has no handler, so delivery attempts fail until MaxOutboxRetryAge elapses. + // The receiver has no handler, so delivery attempts fail until OutboxRetryTimeout elapses. await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); var (state, deadLetters) = await OnGrainAsync(sender, context => ( Grain(context).GetState(), @@ -117,7 +117,7 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe Assert.Equal(payload, letter.Message.Envelope.Payload); Assert.Equal(SourceSinkGrain.Subject, letter.Message.Envelope.Subject); Assert.True(letter.AttemptCount > 1); - // Each attempt materializes the message, as does a final wake which finds the retry period expired. + // Each attempt materializes the message, as does a final wake which finds the retry timeout expired. Assert.InRange(Probe.For(sender).Materializations, letter.AttemptCount + 1, letter.AttemptCount + 2); // The dead letter keeps its own copy for diagnosis. diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs index 44848e2b60f..37d3abb2568 100644 --- a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs @@ -69,10 +69,10 @@ public async Task EmptyPipeline_DeliversDirectly() } [Fact] - public async Task Backpressured_RetriesBeyondTheRetryAgeUntilTheReceiverTakesTheMessage() + public async Task Backpressured_RetriesBeyondTheRetryTimeoutUntilTheReceiverTakesTheMessage() { - var retryAge = TimeSpan.FromMilliseconds(100); - await using var fixture = await StartAsync(Scripted, options => options.MaxOutboxRetryAge = retryAge); + var retryTimeout = TimeSpan.FromMilliseconds(100); + await using var fixture = await StartAsync(Scripted, options => options.OutboxRetryTimeout = retryTimeout); const int BackpressuredAttempts = 8; fixture.Strategy.Script(Enumerable.Repeat(Step.Answer(DeliveryResult.Backpressured()), BackpressuredAttempts).ToArray()); var elapsed = Stopwatch.StartNew(); @@ -80,16 +80,16 @@ public async Task Backpressured_RetriesBeyondTheRetryAgeUntilTheReceiverTakesThe var (sender, receiver) = await SendAsync(fixture, "backpressured"); var delivered = await fixture.WaitForEffectCountAsync(receiver, 1); - Assert.True(elapsed.Elapsed > retryAge); + Assert.True(elapsed.Elapsed > retryTimeout); Assert.Equal("backpressured", Assert.Single(delivered.Effects).Value); Assert.Equal(BackpressuredAttempts + 1, fixture.Strategy.Executions); Assert.Empty((await fixture.WaitForOutboxCountAsync(sender, 0)).OutboxDeadLetters); } [Fact] - public async Task UnansweredAttempts_DeadLetterAfterMaxOutboxRetryAge() + public async Task UnansweredAttempts_DeadLetterAfterOutboxRetryTimeout() { - await using var fixture = await StartAsync(Scripted, static options => options.MaxOutboxRetryAge = TimeSpan.FromMilliseconds(100)); + await using var fixture = await StartAsync(Scripted, static options => options.OutboxRetryTimeout = TimeSpan.FromMilliseconds(100)); fixture.Strategy.Repeat(Step.Unanswered()); var (sender, receiver) = await SendAsync(fixture, "unanswered"); @@ -101,14 +101,14 @@ public async Task UnansweredAttempts_DeadLetterAfterMaxOutboxRetryAge() } [Fact] - public async Task Backpressured_RestartsTheRetryAge() + public async Task Backpressured_RestartsTheRetryTimeout() { - // Each failing span stays within the retry age, while both spans together exceed it. + // Each failing span stays within the retry timeout, while both spans together exceed it. var timeout = TimeSpan.FromMilliseconds(1600); await using var fixture = await StartAsync(Scripted, static options => { - options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3); - options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(1); + options.OutboxRetryTimeout = TimeSpan.FromSeconds(3); + options.RetryInitialInterval = TimeSpan.FromMilliseconds(1); }); fixture.Strategy.Script(Step.Unanswered(timeout), Step.Answer(DeliveryResult.Backpressured()), Step.Answer(DeliveryResult.HandlerNotFound(), timeout)); @@ -121,9 +121,9 @@ public async Task Backpressured_RestartsTheRetryAge() } [Fact] - public async Task HandlerNotFound_DeadLettersAfterMaxOutboxRetryAge() + public async Task HandlerNotFound_DeadLettersAfterOutboxRetryTimeout() { - await using var fixture = await StartAsync(Scripted, static options => options.MaxOutboxRetryAge = TimeSpan.FromSeconds(3)); + await using var fixture = await StartAsync(Scripted, static options => options.OutboxRetryTimeout = TimeSpan.FromSeconds(3)); fixture.Strategy.Repeat(Step.Answer(DeliveryResult.HandlerNotFound())); var (sender, receiver) = await SendAsync(fixture, "unhandled"); @@ -168,7 +168,7 @@ private sealed class DeliveryResilienceClusterFixture( protected override void ConfigureOptions(DurableMessagingOptions options) { base.ConfigureOptions(options); - options.BackpressureRetryDelay = TimeSpan.FromMilliseconds(5); + options.RetryInitialInterval = TimeSpan.FromMilliseconds(5); options.DeliveryResiliencePipeline = createPipeline(Strategy); configure?.Invoke(options); } From 721ab53648d00fbffcdae0f3bd39caff0a2bf682 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 14 Oct 2026 15:19:09 -0700 Subject: [PATCH 79/95] feat(messaging): keep activations alive while outbox deliveries are in flight The messaging pump holds the activation while a pass runs over inbox or outbox work, and keeps holding it while outbox delivery attempts are in flight, so idle collection cannot interrupt a delivery and lose its outcome. --- src/Orleans.Messaging/MessagingPump.cs | 6 +++--- src/Orleans.Messaging/Outbox.cs | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 9daee586482..4a87311c78b 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -19,7 +19,7 @@ namespace Orleans.Messaging; /// earlier wake-up supersedes completes when it runs. /// Wake-ups may be early, late, or duplicated: every pass derives the work to perform from current state. /// Wake-up bookkeeping runs on the activation scheduler: journal writes prepare on it and serialize with job runs. -/// The loop keeps the activation alive while a pass runs over queued work. +/// The loop keeps the activation alive while a pass runs over queued work and while outbox deliveries are in flight. /// Work which only becomes due later relies on the durable job, so the activation can be collected while it waits. /// internal sealed partial class MessagingPump : IDurableJobFeatureHandler, ILifecycleObserver @@ -224,7 +224,7 @@ private async Task RunAsync() var cancellationToken = _shutdown.Token; while (!cancellationToken.IsCancellationRequested) { - SetKeepAlive(Inbox.Count > 0); + SetKeepAlive(Inbox.Count > 0 || Outbox.Count > 0); var pass = _nextPass; _nextPass = null; var progress = false; @@ -247,7 +247,7 @@ private async Task RunAsync() pass?.TrySetResult(); if (!progress) { - SetKeepAlive(false); + SetKeepAlive(Outbox.IsDelivering); await _signal.WaitAsync().ConfigureAwait(true); } } diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 005034ce906..bb6be09d75d 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -44,6 +44,9 @@ public Outbox(MessagingPump pump) internal IReadOnlyDictionary DeadLetterState => _deadLetters; internal int Count => _messages.Count; + /// Gets whether delivery attempts are in flight or awaiting application by the pump. + internal bool IsDelivering => _deliveries is not null; + public IOutbox Register(IOutboxMessageSource source) { ArgumentNullException.ThrowIfNull(source); From 9eff6acc160dec0e30302acacd3dd7e95268250c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 05:35:49 -0700 Subject: [PATCH 80/95] feat(messaging)!: describe outbox retirements with a status and the dead letter's reason OutboxMessageRetirement is a struct which carries the retirement's status and reason, so the contract can grow without changing OnRetired's signature. Its OutboxMessageRetirementStatus distinguishes a message which the receiver rejected from one which the outbox dead-lettered, and Reason carries the dead letter's reason, such as the rejection which the receiver reported. The outbox retires a reference entry to its source as Rejected with the receiver's reason, or as DeadLettered with the dead letter's reason. --- src/Orleans.Messaging/IOutboxMessageSource.cs | 48 ++++++++++++++++--- src/Orleans.Messaging/Outbox.cs | 8 ++-- src/Orleans.Messaging/README.md | 7 +-- .../Orleans.Messaging/Orleans.Messaging.cs | 20 +++++++- .../Functional/MessageSourceAndSinkTests.cs | 41 ++++++++++------ 5 files changed, 95 insertions(+), 29 deletions(-) diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index 05b338de1bb..8c047d4b256 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -28,7 +28,7 @@ namespace Orleans.Messaging; /// A source which can't provide a pending message after its send, for example because its stored payload no longer /// decodes, throws from ; an invalid message is handled the same way. The outbox then /// dead-letters that entry alone, without delivering it, and calls with -/// ; the grain's messaging and its other messages are unaffected. Because no valid message exists, the dead letter's message carries only its +/// ; the grain's messaging and its other messages are unaffected. Because no valid message exists, the dead letter's message carries only its /// , with an empty payload and a default , and its /// reason records the failure. /// @@ -55,13 +55,13 @@ public interface IOutboxMessageSource /// message fails the send, and nothing is staged. Afterwards, the entry is durable: an exception or an invalid message, /// including one with a different or addressed to the sending grain, dead-letters the /// entry with its failure as the reason, counting the failed materialization as a delivery attempt, and the outbox - /// then calls with . + /// then calls with . /// bool TryGetMessage(MessageId messageId, out OutboxMessage message); /// Notifies the source that the outbox retired a message, so the source can release its payload. /// The ID of the retired reference entry. - /// Whether the receiver retired the message or the message was dead-lettered. + /// How the message retired, and the dead letter's reason for a dead-lettered message. /// /// Stage the release synchronously in journaled state; it commits in the journal write which removes the entry. /// A dead-lettered message is materialized into the outbox dead letter before this call, unless the source failed to @@ -72,14 +72,48 @@ public interface IOutboxMessageSource } /// Describes how an outbox message retired. -public enum OutboxMessageRetirement +/// The outbox creates retirements; sources read them in . +public readonly struct OutboxMessageRetirement +{ + /// Gets how the message retired. + public OutboxMessageRetirementStatus Status { get; init; } + + /// + /// Gets the dead letter's reason for a message which the receiver rejected or the outbox dead-lettered, such as the + /// rejection which the receiver reported; for a delivered message. + /// + public string? Reason { get; init; } + + /// Creates a retirement of a message which the receiver durably accepted or reported as a duplicate. + public static OutboxMessageRetirement Delivered() => new() { Status = OutboxMessageRetirementStatus.Delivered }; + + /// Creates a retirement of a message which the outbox dead-lettered for . + public static OutboxMessageRetirement DeadLettered(string reason) => new() + { + Status = OutboxMessageRetirementStatus.DeadLettered, + Reason = reason + }; + + /// Creates a retirement of a message which the receiver dead-lettered for . + public static OutboxMessageRetirement Rejected(string reason) => new() + { + Status = OutboxMessageRetirementStatus.Rejected, + Reason = reason + }; +} + +/// Describes how an outbox message retired. +public enum OutboxMessageRetirementStatus { /// The receiver durably accepted the message, or reported it as a duplicate. Delivered = 1, /// - /// The receiver dead-lettered the message, or the outbox dead-lettered it after its retry limits elapsed or because - /// its source failed to provide it. + /// The outbox dead-lettered the message: its delivery attempts didn't succeed before its retry limits elapsed, or its + /// source failed to provide it. /// - DeadLettered = 2 + DeadLettered = 2, + + /// The receiver dead-lettered the message, with the reason which it reported. + Rejected = 3 } diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index bb6be09d75d..9ae10acf3a3 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -436,7 +436,9 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) if (status is DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.DeadLettered) { Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id }); - Retire(current, status == DeliveryStatus.DeadLettered ? OutboxMessageRetirement.DeadLettered : OutboxMessageRetirement.Delivered); + Retire(current, status == DeliveryStatus.DeadLettered + ? OutboxMessageRetirement.Rejected(outcome.Result!.Value.Message ?? "The receiver dead-lettered the message.") + : OutboxMessageRetirement.Delivered()); delivered++; } else @@ -444,7 +446,7 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) if (status == DeliveryStatus.Backpressured) backpressured++; var (retry, deadLetter) = PrepareRetry(outcome, ref deadLetters); Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id, State = retry, DeadLetter = deadLetter }); - if (deadLetter is not null) Retire(current, OutboxMessageRetirement.DeadLettered); + if (deadLetter is not null) Retire(current, OutboxMessageRetirement.DeadLettered(deadLetter.Reason)); } RecordDeliveryMetrics(outcome); applied++; @@ -520,7 +522,7 @@ private void DeadLetterSourceFailures(List<(OutboxEntry Entry, Exception Error)> }; AddDeadLetter(ref deadLetters, letter); Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id, DeadLetter = letter }); - Retire(entry, OutboxMessageRetirement.DeadLettered); + Retire(entry, OutboxMessageRetirement.DeadLettered(letter.Reason)); Shared.Instruments.OnOutboxMessageDelivered(Shared.GrainType, "source_error"); } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 7b5e42ad1c4..8d1db875f3c 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -116,9 +116,10 @@ construction or activation setup, before journal recovery. delivery rule above applies. A source withdraws a pending message by returning `false`: the outbox removes the entry before its next attempt, without delivery or `OnRetired`, and an attempt already in flight still completes. When the message - retires, the outbox calls `OnRetired` with - `OutboxMessageRetirement.Delivered` (`Accepted` or `Duplicate`) or `DeadLettered` - (receiver-reported or outbox dead-lettering) while it stages the removal, so the + retires, the outbox calls `OnRetired` with an `OutboxMessageRetirement` whose + `Status` is `Delivered` (`Accepted` or `Duplicate`), `Rejected` (the receiver + dead-lettered it) or `DeadLettered` (the outbox dead-lettered it), and whose + `Reason` carries the dead letter's reason, while it stages the removal, so the source's changes commit in the same journal write. An outbox dead letter stores the materialized message. After the send, a source that throws or returns an invalid message, such as one with a different ID, dead-letters that entry alone in the diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 8bb13e751b7..9f5c993fbd9 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -345,10 +345,26 @@ public readonly partial struct OutboxMessage public readonly InboxMessage ToInboxMessage() { throw null; } } - public enum OutboxMessageRetirement + public readonly partial struct OutboxMessageRetirement + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public string? Reason { get { throw null; } init { } } + + public OutboxMessageRetirementStatus Status { get { throw null; } init { } } + + public static OutboxMessageRetirement DeadLettered(string reason) { throw null; } + + public static OutboxMessageRetirement Delivered() { throw null; } + + public static OutboxMessageRetirement Rejected(string reason) { throw null; } + } + + public enum OutboxMessageRetirementStatus { Delivered = 1, - DeadLettered = 2 + DeadLettered = 2, + Rejected = 3 } } diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 3653911a06e..6cda3e47e5d 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -39,7 +39,8 @@ public async Task SourceMessage_JournalsEachPayloadOnceAndRetiresInTheRemovalWri await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Writes(sender).Length == 2); var senderState = await OnGrainAsync(sender, context => Grain(context).GetState()); Assert.Empty(senderState.Outgoing); - Assert.Equal(nameof(OutboxMessageRetirement.Delivered), Assert.Single(senderState.Retired).Value); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), Assert.Single(senderState.Retired).Value); + Assert.Empty(await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons())); // The outbox materializes the message once to validate the send and once for the delivery attempt. Assert.Equal(2, Probe.For(sender).Materializations); @@ -49,7 +50,7 @@ public async Task SourceMessage_JournalsEachPayloadOnceAndRetiresInTheRemovalWri // The retirement commits in the write which removes the outbox entry. var writes = Writes(sender); - var retirement = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirement.Delivered)); + var retirement = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirementStatus.Delivered)); Assert.True(writes[0].AsSpan().IndexOf(retirement) < 0); Assert.True(writes[1].AsSpan().IndexOf(retirement) >= 0); @@ -110,8 +111,9 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe Grain(context).GetState(), context.ActivationServices.GetRequiredService().OutboxDeadLetters)); Assert.Empty(state.Outgoing); - Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[messageId.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[messageId.ToString()]); var letter = Assert.Single(deadLetters); + Assert.Equal(letter.Reason, (await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons()))[messageId.ToString()]); Assert.Equal(messageId, letter.Message.Envelope.MessageId); Assert.Equal(receiver.GetGrainId(), letter.Message.ReceiverId); Assert.Equal(payload, letter.Message.Envelope.Payload); @@ -125,7 +127,7 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe } [Fact] - public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsDeadLettered() + public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithTheReceiversReason() { var sender = NewGrain(); var receiver = NewGrain(); @@ -139,7 +141,8 @@ public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsDeadLettered( var (state, outboxDeadLetters) = await OnGrainAsync(sender, context => ( Grain(context).GetState(), context.ActivationServices.GetRequiredService().OutboxDeadLetters)); - Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[messageId.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Rejected), state.Retired[messageId.ToString()]); + Assert.Equal(SourceSinkGrain.Rejection, (await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons()))[messageId.ToString()]); Assert.Empty(state.Outgoing); Assert.Empty(outboxDeadLetters); var inboxLetter = Assert.Single(await OnGrainAsync(receiver, context => @@ -227,18 +230,21 @@ public async Task SourceFailureAfterSend_DeadLettersOnlyThatEntryAndKeepsTheGrai Grain(context).GetState(), context.ActivationServices.GetRequiredService().OutboxDeadLetters)); Assert.Empty(state.Outgoing); - Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[failing.ToString()]); - Assert.Equal(nameof(OutboxMessageRetirement.Delivered), state.Retired[first.ToString()]); - Assert.Equal(nameof(OutboxMessageRetirement.Delivered), state.Retired[second.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[failing.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), state.Retired[first.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), state.Retired[second.ToString()]); var received = (await OnGrainAsync(receiver, context => Grain(context).GetState())).Received; Assert.Equal(new[] { first.ToString(), second.ToString() }.Order(), received.Keys.Order()); AssertSourceFailureDeadLetter(Assert.Single(deadLetters), failing, fault); + Assert.Equal( + Assert.Single(deadLetters).Reason, + Assert.Single(await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons())).Value); // The dead letter and its retirement commit in one write, before the write which retires the delivered siblings. var writes = Writes(sender); Assert.Equal(3, writes.Length); - var deadLettered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirement.DeadLettered)); - var delivered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirement.Delivered)); + var deadLettered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirementStatus.DeadLettered)); + var delivered = Encoding.UTF8.GetBytes(nameof(OutboxMessageRetirementStatus.Delivered)); Assert.True(writes[1].AsSpan().IndexOf(deadLettered) >= 0); Assert.True(writes[1].AsSpan().IndexOf(delivered) < 0); Assert.True(writes[2].AsSpan().IndexOf(deadLettered) < 0); @@ -254,7 +260,7 @@ public async Task SourceFailureAfterSend_DeadLettersOnlyThatEntryAndKeepsTheGrai var later = NewMessageId("later"); await sender.StageFromSourceAsync(receiver.GetGrainId(), later, NewPayload(), write: true); await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 4); - Assert.Equal(nameof(OutboxMessageRetirement.Delivered), (await OnGrainAsync(sender, context => Grain(context).GetState())).Retired[later.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), (await OnGrainAsync(sender, context => Grain(context).GetState())).Retired[later.ToString()]); Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var current)); Assert.Same(activation, current); Assert.Equal(1, Probe.For(sender).Activations); @@ -298,14 +304,14 @@ await OnGrainAsync(grain, context => Grain(context).GetState(), context.ActivationServices.GetRequiredService().OutboxDeadLetters)); Assert.Empty(state.Outgoing); - Assert.Equal(nameof(OutboxMessageRetirement.DeadLettered), state.Retired[messageId.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[messageId.ToString()]); AssertSourceFailureDeadLetter(Assert.Single(deadLetters), messageId, SourceFault.Throw); // The activation keeps processing messages. var later = NewMessageId("after-recovered-failure"); await grain.StageFromSourceAsync(receiver, later, NewPayload(), write: true); await WaitUntilAsync(grain, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 2); - Assert.Equal(nameof(OutboxMessageRetirement.Delivered), (await OnGrainAsync(grain, context => Grain(context).GetState())).Retired[later.ToString()]); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), (await OnGrainAsync(grain, context => Grain(context).GetState())).Retired[later.ToString()]); Assert.Equal(1, Probe.For(grain).Activations); } @@ -813,6 +819,7 @@ public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, private readonly IDurableDictionary _receivers; private readonly IDurableDictionary _received; private readonly IDurableDictionary _retired; + private readonly IDurableDictionary _reasons; private readonly SourceSinkProbe.Counters _probe; public SourceSinkGrain( @@ -824,6 +831,7 @@ public SourceSinkGrain( [FromKeyedServices("receivers")] IDurableDictionary receivers, [FromKeyedServices("received")] IDurableDictionary received, [FromKeyedServices("retired")] IDurableDictionary retired, + [FromKeyedServices("reasons")] IDurableDictionary reasons, SourceSinkProbe probe) { _manager = manager; @@ -832,6 +840,7 @@ public SourceSinkGrain( _receivers = receivers; _received = received; _retired = retired; + _reasons = reasons; _probe = probe.For(context.GrainId); _probe.OnActivated(); outbox.Register(this); @@ -841,6 +850,9 @@ public SourceSinkGrain( internal (Dictionary Outgoing, Dictionary Received, Dictionary Retired) GetState() => (new(_outgoing), new(_received), new(_retired)); + /// Gets the reasons of this source's dead-lettered messages, by message ID. + internal Dictionary GetRetirementReasons() => new(_reasons); + /// Stores an outgoing message in the source's state without sending it. internal void StoreOutgoing(GrainId receiver, MessageId messageId, byte[] payload) { @@ -928,7 +940,8 @@ void IOutboxMessageSource.OnRetired(MessageId messageId, OutboxMessageRetirement var key = messageId.ToString(); Assert.True(_outgoing.Remove(key)); Assert.True(_receivers.Remove(key)); - _retired[key] = retirement.ToString(); + _retired[key] = retirement.Status.ToString(); + if (retirement.Reason is { } reason) _reasons[key] = reason; } bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result) From 85cb63c383f56e69f0e6611324fcd7997d07f068 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 06:45:20 -0700 Subject: [PATCH 81/95] feat(messaging): carry an application-defined code with sink rejections An inbox message sink can dead-letter a message with an opaque code next to the reason. The inbox records it on the dead letter and returns it to the sender, whose outbox reports it as OutboxMessageRetirement.Code on a Rejected retirement. Sink exceptions keep dead-lettering with the exception message and no code. --- src/Orleans.Messaging/DeliveryResult.cs | 18 +++++++++-- src/Orleans.Messaging/IInboxMessageSink.cs | 7 +++-- .../IMessagingDiagnostics.cs | 10 +++++- src/Orleans.Messaging/IOutboxMessageSource.cs | 14 +++++++-- src/Orleans.Messaging/Inbox.State.cs | 1 + src/Orleans.Messaging/Inbox.cs | 11 ++++--- src/Orleans.Messaging/MessageState.cs | 3 ++ src/Orleans.Messaging/Outbox.cs | 2 +- src/Orleans.Messaging/README.md | 15 +++++---- .../Orleans.Messaging/Orleans.Messaging.cs | 12 +++++-- .../DeliveryAndOptionsContractTests.cs | 5 +++ .../Contracts/EnvelopeContractTests.cs | 6 ++-- .../MessagingCommandSerializationTests.cs | 4 +-- .../Functional/MessageSourceAndSinkTests.cs | 31 ++++++++++++++----- 14 files changed, 105 insertions(+), 34 deletions(-) diff --git a/src/Orleans.Messaging/DeliveryResult.cs b/src/Orleans.Messaging/DeliveryResult.cs index 6124b87548c..efb37334373 100644 --- a/src/Orleans.Messaging/DeliveryResult.cs +++ b/src/Orleans.Messaging/DeliveryResult.cs @@ -20,6 +20,17 @@ public readonly struct DeliveryResult [Id(1)] public string? Message { get; init; } + /// + /// Gets the application-defined code which identifies why an inbox message sink dead-lettered the message, such as + /// "orders/duplicate-order"; when the sink reported none. + /// + /// + /// Messaging carries the code to the inbox dead letter and to the sender's + /// without interpreting it. + /// + [Id(2)] + public string? Code { get; init; } + /// /// Creates a result indicating the message was accepted and persisted to inbox. /// @@ -47,9 +58,12 @@ public readonly struct DeliveryResult /// /// Creates a result indicating the message was dead-lettered. /// - public static DeliveryResult DeadLettered(string reason) => new() + /// The diagnostic reason. + /// The application-defined code which identifies the rejection, if any. + public static DeliveryResult DeadLettered(string reason, string? code = null) => new() { Status = DeliveryStatus.DeadLettered, - Message = reason + Message = reason, + Code = code }; } diff --git a/src/Orleans.Messaging/IInboxMessageSink.cs b/src/Orleans.Messaging/IInboxMessageSink.cs index fd633303641..9c104675d64 100644 --- a/src/Orleans.Messaging/IInboxMessageSink.cs +++ b/src/Orleans.Messaging/IInboxMessageSink.cs @@ -57,13 +57,14 @@ public interface IInboxMessageSink /// /// /// : the sink rejects the message and stages nothing. The inbox - /// dead-letters it with its payload and the result's reason, and records it as processed. The processed record keeps - /// the reason, so a duplicate of the message, such as a retry whose first answer was lost, receives the same rejection. + /// dead-letters it with its payload and the result's reason and , records it as + /// processed, and returns the reason and code to the sender. The processed record keeps the reason and code, so a + /// duplicate of the message, such as a retry whose first answer was lost, receives the same rejection. /// /// /// /// A sink which throws or reports another status must leave no staged change, and the inbox dead-letters the message - /// with the failure as its reason. + /// with the failure as its reason and no code. /// /// bool TryAccept(InboxMessage message, out DeliveryResult result); diff --git a/src/Orleans.Messaging/IMessagingDiagnostics.cs b/src/Orleans.Messaging/IMessagingDiagnostics.cs index f255507296a..a7f462161ed 100644 --- a/src/Orleans.Messaging/IMessagingDiagnostics.cs +++ b/src/Orleans.Messaging/IMessagingDiagnostics.cs @@ -80,6 +80,13 @@ public sealed class DeadLetter /// [Id(3)] public int AttemptCount { get; init; } + + /// + /// Gets the application-defined code which an inbox message sink reported when it dead-lettered the message, or + /// . + /// + [Id(4)] + public string? Code { get; init; } } internal sealed class MessagingDiagnostics(Inbox inbox, Outbox outbox) : IMessagingDiagnostics @@ -90,7 +97,8 @@ internal sealed class MessagingDiagnostics(Inbox inbox, Outbox outbox) : IMessag Message = entry.Message, DeadLetteredAt = entry.DeadLetteredAt, Reason = entry.Reason, - AttemptCount = entry.AttemptCount + AttemptCount = entry.AttemptCount, + Code = entry.Code }).ToList(); public IReadOnlyList> OutboxDeadLetters => diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index 8c047d4b256..476943ee3da 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -84,6 +84,13 @@ public readonly struct OutboxMessageRetirement /// public string? Reason { get; init; } + /// + /// Gets the application-defined code which the receiver's inbox message sink reported with a rejection, as + /// ; when the receiver reported none or the message didn't + /// retire as . + /// + public string? Code { get; init; } + /// Creates a retirement of a message which the receiver durably accepted or reported as a duplicate. public static OutboxMessageRetirement Delivered() => new() { Status = OutboxMessageRetirementStatus.Delivered }; @@ -95,10 +102,13 @@ public readonly struct OutboxMessageRetirement }; /// Creates a retirement of a message which the receiver dead-lettered for . - public static OutboxMessageRetirement Rejected(string reason) => new() + /// The reason which the receiver reported. + /// The application-defined code which the receiver reported, if any. + public static OutboxMessageRetirement Rejected(string reason, string? code = null) => new() { Status = OutboxMessageRetirementStatus.Rejected, - Reason = reason + Reason = reason, + Code = code }; } diff --git a/src/Orleans.Messaging/Inbox.State.cs b/src/Orleans.Messaging/Inbox.State.cs index 7b113143f85..ea978c8ae94 100644 --- a/src/Orleans.Messaging/Inbox.State.cs +++ b/src/Orleans.Messaging/Inbox.State.cs @@ -252,6 +252,7 @@ internal sealed class InboxProcessedEntry internal sealed class InboxRejection { [Id(0)] public required string Reason { get; init; } + [Id(1)] public string? Code { get; init; } } [GenerateSerializer, Alias("Orleans.Messaging.InboxEntry")] diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 240558de6a9..738b532a290 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -94,7 +94,7 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel // A sender whose answer was lost delivers again, so it receives the rejection which a sink first answered. return pending is null && _rejections.TryGetValue(key, out var rejection) - ? DeliveryResult.DeadLettered(rejection.Reason) + ? DeliveryResult.DeadLettered(rejection.Reason, rejection.Code) : DeliveryResult.Duplicate(); } @@ -236,7 +236,7 @@ internal void StageLocal(InboxMessage message) else { LogSinkDeadLettered(Logger, error, key, GrainContext.GrainId, reason); - StageDeadLetter(message, reason, attemptCount: 1, new() { Reason = reason }); + StageDeadLetter(message, reason, attemptCount: 1, result.Code, new() { Reason = reason, Code = result.Code }); } } catch (Exception exception) @@ -249,7 +249,7 @@ internal void StageLocal(InboxMessage message) Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, accepted ? "accepted" : "dead_lettered"); Shared.Instruments.OnInboxMessageProcessed(Shared.GrainType, accepted ? "success" : "dead_lettered"); - if (!accepted) return DeliveryResult.DeadLettered(reason); + if (!accepted) return DeliveryResult.DeadLettered(reason, result.Code); LogMessageAccepted(Logger, key, GrainContext.GrainId); return DeliveryResult.Accepted(); } @@ -426,7 +426,7 @@ private void StageCompletion(MessageId messageId, InboxDeadLetter? deadLetter, I Rejection = rejection }); - private void StageDeadLetter(InboxMessage message, string reason, int attemptCount, InboxRejection? rejection = null) + private void StageDeadLetter(InboxMessage message, string reason, int attemptCount, string? code = null, InboxRejection? rejection = null) { var now = TimeProvider.GetUtcNow(); var key = message.Envelope.MessageId; @@ -436,7 +436,8 @@ private void StageDeadLetter(InboxMessage message, string reason, int attemptCou Message = message, DeadLetteredAt = now, Reason = reason, - AttemptCount = attemptCount + AttemptCount = attemptCount, + Code = code }, rejection); } diff --git a/src/Orleans.Messaging/MessageState.cs b/src/Orleans.Messaging/MessageState.cs index 3a055ce7e01..9621835c653 100644 --- a/src/Orleans.Messaging/MessageState.cs +++ b/src/Orleans.Messaging/MessageState.cs @@ -54,6 +54,9 @@ internal sealed class InboxDeadLetter [Id(3)] public int AttemptCount { get; init; } + + [Id(4)] + public string? Code { get; init; } } [GenerateSerializer, Alias("Orleans.Messaging.OutboxDeadLetter")] diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 9ae10acf3a3..d033f2e5174 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -437,7 +437,7 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) { Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id }); Retire(current, status == DeliveryStatus.DeadLettered - ? OutboxMessageRetirement.Rejected(outcome.Result!.Value.Message ?? "The receiver dead-lettered the message.") + ? OutboxMessageRetirement.Rejected(outcome.Result!.Value.Message ?? "The receiver dead-lettered the message.", outcome.Result.Value.Code) : OutboxMessageRetirement.Delivered()); delivered++; } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 8d1db875f3c..6696586d98b 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -118,8 +118,9 @@ construction or activation setup, before journal recovery. `OnRetired`, and an attempt already in flight still completes. When the message retires, the outbox calls `OnRetired` with an `OutboxMessageRetirement` whose `Status` is `Delivered` (`Accepted` or `Duplicate`), `Rejected` (the receiver - dead-lettered it) or `DeadLettered` (the outbox dead-lettered it), and whose - `Reason` carries the dead letter's reason, while it stages the removal, so the + dead-lettered it) or `DeadLettered` (the outbox dead-lettered it), whose + `Reason` carries the dead letter's reason, and whose `Code` carries the code of a + `Rejected` message's receiver, while it stages the removal, so the source's changes commit in the same journal write. An outbox dead letter stores the materialized message. After the send, a source that throws or returns an invalid message, such as one with a different ID, dead-letters that entry alone in the @@ -141,10 +142,12 @@ construction or activation setup, before journal recovery. - `Duplicate`: the sink already holds the message. - `Backpressured`: the sink is at capacity, so the sender retries. A self-sent message fails its `Send` instead. - - `DeadLettered`: the inbox dead-letters the message with the sink's reason. A - sink that throws or reports another status is treated the same way. The processed - record keeps the reason, so a duplicate within `DeduplicationWindow`, such as a - retry whose first answer was lost, receives the same rejection. + - `DeadLettered`: the inbox dead-letters the message with the sink's reason and + optional application-defined `Code`, such as `orders/duplicate-order`, and the + sender's source receives both in a `Rejected` retirement. A sink that throws or + reports another status is treated the same way, with no code. The processed + record keeps the reason and code, so a duplicate within `DeduplicationWindow`, + such as a retry whose first answer was lost, receives the same rejection. Sink-owned messages never occupy capacity and have no inbox retry lifecycle; the sink honors their `DeliverAt`. Declined messages follow the handler path once diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 9f5c993fbd9..1068f79650c 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -30,6 +30,9 @@ public sealed partial class DeadLetter [Id(3)] public int AttemptCount { get { throw null; } init { } } + [Id(4)] + public string? Code { get { throw null; } init { } } + [Id(1)] public System.DateTimeOffset DeadLetteredAt { get { throw null; } init { } } @@ -46,6 +49,9 @@ public readonly partial struct DeliveryResult { private readonly object _dummy; private readonly int _dummyPrimitive; + [Id(2)] + public string? Code { get { throw null; } init { } } + [Id(1)] public string? Message { get { throw null; } init { } } @@ -56,7 +62,7 @@ public readonly partial struct DeliveryResult public static DeliveryResult Backpressured() { throw null; } - public static DeliveryResult DeadLettered(string reason) { throw null; } + public static DeliveryResult DeadLettered(string reason, string? code = null) { throw null; } public static DeliveryResult Duplicate() { throw null; } @@ -349,6 +355,8 @@ public readonly partial struct OutboxMessageRetirement { private readonly object _dummy; private readonly int _dummyPrimitive; + public string? Code { get { throw null; } init { } } + public string? Reason { get { throw null; } init { } } public OutboxMessageRetirementStatus Status { get { throw null; } init { } } @@ -357,7 +365,7 @@ public readonly partial struct OutboxMessageRetirement public static OutboxMessageRetirement Delivered() { throw null; } - public static OutboxMessageRetirement Rejected(string reason) { throw null; } + public static OutboxMessageRetirement Rejected(string reason, string? code = null) { throw null; } } public enum OutboxMessageRetirementStatus diff --git a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs index 530f7c8e593..e3ce4c9c336 100644 --- a/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/DeliveryAndOptionsContractTests.cs @@ -45,6 +45,7 @@ public void DeliveryResult_EachFactory_PreservesStatusAndPayload() { var handlerMissing = DeliveryResult.HandlerNotFound(); var deadLettered = DeliveryResult.DeadLettered("poison body"); + var coded = DeliveryResult.DeadLettered("poison body", "orders/poison"); Assert.Equal(DeliveryStatus.Accepted, DeliveryResult.Accepted().Status); Assert.Equal(DeliveryStatus.Duplicate, DeliveryResult.Duplicate().Status); @@ -53,6 +54,10 @@ public void DeliveryResult_EachFactory_PreservesStatusAndPayload() Assert.Equal("No inbox handler is registered.", handlerMissing.Message); Assert.Equal(DeliveryStatus.DeadLettered, deadLettered.Status); Assert.Equal("poison body", deadLettered.Message); + Assert.Null(deadLettered.Code); + Assert.Equal(DeliveryStatus.DeadLettered, coded.Status); + Assert.Equal("poison body", coded.Message); + Assert.Equal("orders/poison", coded.Code); } [Fact] diff --git a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs index ff64e0ef3bb..7e1d6876847 100644 --- a/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/EnvelopeContractTests.cs @@ -212,7 +212,8 @@ public void PublicDeadLetterRoundTripsThroughOrleansSerialization() Message = new InboxMessage { Envelope = Create([1, 2]) with { Subject = "subject.v1" } }, DeadLetteredAt = DateTimeOffset.UnixEpoch, Reason = "failure", - AttemptCount = 3 + AttemptCount = 3, + Code = "orders/invalid" }; var serializer = services.GetRequiredService>>(); @@ -224,6 +225,7 @@ public void PublicDeadLetterRoundTripsThroughOrleansSerialization() Assert.Equal(DateTimeOffset.UnixEpoch, result.DeadLetteredAt); Assert.Equal("failure", result.Reason); Assert.Equal(3, result.AttemptCount); + Assert.Equal("orders/invalid", result.Code); } [Fact] @@ -245,7 +247,7 @@ public void SimpleEnvelopeSurfaceUsesGeneratedOrdinaryFields() Assert.Equal(typeof(string), Assert.Single(typeof(IInboxHandlerContext).GetMethod("Fail")!.GetParameters()).ParameterType); Assert.Equal(new[] { "Accepted:1", "Backpressured:3", "DeadLettered:5", "Duplicate:2", "HandlerNotFound:4" }, Enum.GetNames().Select(name => $"{name}:{(int)Enum.Parse(name)}").Order(StringComparer.Ordinal)); - AssertIds(typeof(DeliveryResult), ("Status", 0u), ("Message", 1u)); + AssertIds(typeof(DeliveryResult), ("Status", 0u), ("Message", 1u), ("Code", 2u)); using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); var outgoing = new OutboxMessage { Envelope = Create([7]), ReceiverId = Receiver }; var other = outgoing with { ReceiverId = GrainId.Create("receiver", "other") }; diff --git a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs index 5bcf37eb89b..41e292a5802 100644 --- a/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs +++ b/test/Orleans.Messaging.Tests/Contracts/MessagingCommandSerializationTests.cs @@ -113,8 +113,8 @@ public void ReferenceEnqueue_CarriesNoMessageInJsonAndOrleansSerialization() private static InboxCommand[] CreateInboxCommands() { - var deadLetter = new InboxDeadLetter { Message = InboxMessage("dead"), DeadLetteredAt = Now, Reason = "failed", AttemptCount = 3 }; - var rejection = new InboxRejection { Reason = "failed" }; + var deadLetter = new InboxDeadLetter { Message = InboxMessage("dead"), DeadLetteredAt = Now, Reason = "failed", AttemptCount = 3, Code = "orders/invalid" }; + var rejection = new InboxRejection { Reason = "failed", Code = "orders/invalid" }; return [ new() { Kind = InboxCommandKind.Accept, Message = InboxMessage("accept"), State = new() }, diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 6cda3e47e5d..7213a478997 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -126,12 +126,14 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 2); } - [Fact] - public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithTheReceiversReason() + [Theory] + [InlineData(SinkMode.Throw, null)] + [InlineData(SinkMode.DeadLetter, SourceSinkGrain.RejectionCode)] + public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithTheReceiversReasonAndCode(SinkMode mode, string? code) { var sender = NewGrain(); var receiver = NewGrain(); - Probe.For(receiver).Mode = SinkMode.Throw; + Probe.For(receiver).Mode = mode; var messageId = NewMessageId("receiver-dead-letter"); var payload = NewPayload(); @@ -143,12 +145,14 @@ public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithT context.ActivationServices.GetRequiredService().OutboxDeadLetters)); Assert.Equal(nameof(OutboxMessageRetirementStatus.Rejected), state.Retired[messageId.ToString()]); Assert.Equal(SourceSinkGrain.Rejection, (await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons()))[messageId.ToString()]); + Assert.Equal(code, (await OnGrainAsync(sender, context => Grain(context).GetRetirementCodes())).GetValueOrDefault(messageId.ToString())); Assert.Empty(state.Outgoing); Assert.Empty(outboxDeadLetters); var inboxLetter = Assert.Single(await OnGrainAsync(receiver, context => context.ActivationServices.GetRequiredService().InboxDeadLetters)); Assert.Equal(payload, inboxLetter.Message.Envelope.Payload); Assert.Equal(SourceSinkGrain.Rejection, inboxLetter.Reason); + Assert.Equal(code, inboxLetter.Code); } [Fact] @@ -465,10 +469,10 @@ public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndTheSinkReportsLat } [Theory] - [InlineData(SinkMode.Throw, SourceSinkGrain.Rejection)] - [InlineData(SinkMode.DeadLetter, SourceSinkGrain.Rejection)] - [InlineData(SinkMode.UnsupportedStatus, "The inbox message sink reported the unsupported delivery status 'HandlerNotFound'.")] - public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWithTheSameRejection(SinkMode mode, string reason) + [InlineData(SinkMode.Throw, SourceSinkGrain.Rejection, null)] + [InlineData(SinkMode.DeadLetter, SourceSinkGrain.Rejection, SourceSinkGrain.RejectionCode)] + [InlineData(SinkMode.UnsupportedStatus, "The inbox message sink reported the unsupported delivery status 'HandlerNotFound'.", null)] + public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWithTheSameRejection(SinkMode mode, string reason, string? code) { var receiver = NewGrain(); Probe.For(receiver).Mode = mode; @@ -482,6 +486,7 @@ public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWith Assert.Equal(DeliveryStatus.DeadLettered, result.Status); Assert.Equal(reason, result.Message); + Assert.Equal(code, result.Code); var grainType = receiver.GetGrainId().Type.ToString(); Assert.Single(metrics.Read("inbox-messages-received")).AssertCounter(grainType, "dead_lettered"); Assert.Single(metrics.Read("inbox-messages-processed")).AssertCounter(grainType, "dead_lettered"); @@ -494,6 +499,7 @@ public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWith var letter = Assert.Single(letters); Assert.Equal(payload, letter.Message.Envelope.Payload); Assert.Equal(reason, letter.Reason); + Assert.Equal(code, letter.Code); Assert.Equal(1, letter.AttemptCount); Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); @@ -518,6 +524,7 @@ void AssertRejected(DeliveryResult duplicate) { Assert.Equal(DeliveryStatus.DeadLettered, duplicate.Status); Assert.Equal(reason, duplicate.Message); + Assert.Equal(code, duplicate.Code); } } @@ -812,6 +819,7 @@ public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, { internal const string Subject = "source-sink/payload"; internal const string Rejection = "The sink rejected the message."; + internal const string RejectionCode = "source-sink/rejected"; internal const string SourceFailure = "The source can't decode the stored message."; private readonly IJournaledStateManager _manager; private readonly IOutbox _outbox; @@ -820,6 +828,7 @@ public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, private readonly IDurableDictionary _received; private readonly IDurableDictionary _retired; private readonly IDurableDictionary _reasons; + private readonly IDurableDictionary _codes; private readonly SourceSinkProbe.Counters _probe; public SourceSinkGrain( @@ -832,6 +841,7 @@ public SourceSinkGrain( [FromKeyedServices("received")] IDurableDictionary received, [FromKeyedServices("retired")] IDurableDictionary retired, [FromKeyedServices("reasons")] IDurableDictionary reasons, + [FromKeyedServices("codes")] IDurableDictionary codes, SourceSinkProbe probe) { _manager = manager; @@ -841,6 +851,7 @@ public SourceSinkGrain( _received = received; _retired = retired; _reasons = reasons; + _codes = codes; _probe = probe.For(context.GrainId); _probe.OnActivated(); outbox.Register(this); @@ -853,6 +864,9 @@ public SourceSinkGrain( /// Gets the reasons of this source's dead-lettered messages, by message ID. internal Dictionary GetRetirementReasons() => new(_reasons); + /// Gets the receivers' rejection codes for this source's rejected messages, by message ID. + internal Dictionary GetRetirementCodes() => new(_codes); + /// Stores an outgoing message in the source's state without sending it. internal void StoreOutgoing(GrainId receiver, MessageId messageId, byte[] payload) { @@ -942,6 +956,7 @@ void IOutboxMessageSource.OnRetired(MessageId messageId, OutboxMessageRetirement Assert.True(_receivers.Remove(key)); _retired[key] = retirement.Status.ToString(); if (retirement.Reason is { } reason) _reasons[key] = reason; + if (retirement.Code is { } code) _codes[key] = code; } bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result) @@ -957,7 +972,7 @@ bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result case SinkMode.Throw: throw new InvalidOperationException(Rejection); case SinkMode.DeadLetter: - result = DeliveryResult.DeadLettered(Rejection); + result = DeliveryResult.DeadLettered(Rejection, RejectionCode); return true; case SinkMode.Backpressure: result = DeliveryResult.Backpressured(); From 7927e43d6a6cf4da5aacf255f4fa311dd1e17b73 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 07:05:56 -0700 Subject: [PATCH 82/95] fix(messaging)!: retry deliveries whose inbox sink throws A sink exception or unsupported status leaves the delivery's outcome unknown, so the inbox stages nothing and fails the delivery instead of dead-lettering the message. The sender's outbox retries it with backoff until OutboxRetryTimeout, and a self-send throws to its caller. Only an explicit DeliveryResult.DeadLettered rejects a message permanently. --- src/Orleans.Messaging/IInboxMessageSink.cs | 6 +- src/Orleans.Messaging/Inbox.cs | 37 ++--- src/Orleans.Messaging/README.md | 12 +- .../Functional/MessageSourceAndSinkTests.cs | 131 +++++++++++++++--- 4 files changed, 142 insertions(+), 44 deletions(-) diff --git a/src/Orleans.Messaging/IInboxMessageSink.cs b/src/Orleans.Messaging/IInboxMessageSink.cs index 9c104675d64..30fda39d469 100644 --- a/src/Orleans.Messaging/IInboxMessageSink.cs +++ b/src/Orleans.Messaging/IInboxMessageSink.cs @@ -63,8 +63,10 @@ public interface IInboxMessageSink /// /// /// - /// A sink which throws or reports another status must leave no staged change, and the inbox dead-letters the message - /// with the failure as its reason and no code. + /// A sink exception, or any other status, means the outcome is unknown, so the inbox stages nothing and fails the + /// delivery: the sender's outbox retries it with backoff until OutboxRetryTimeout, and a send to the grain + /// itself throws to its caller. A sink which throws must leave no staged change. Return + /// to reject a message permanently. /// /// bool TryAccept(InboxMessage message, out DeliveryResult result); diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index 738b532a290..c60feeb13f7 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -153,7 +153,8 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel /// /// /// - /// The inbox holds a different pending command with the same ID, or an inbox sink backpressured the message. + /// The inbox holds a different pending command with the same ID, or an inbox sink backpressured the message or + /// reported an unsupported status. /// internal void StageLocal(InboxMessage message) { @@ -195,36 +196,28 @@ internal void StageLocal(InboxMessage message) /// The delivery result, or when every sink declines the message. /// /// An accepted message journals only its processed record, in the write which commits the sink's staged changes. - /// A duplicate or backpressured message stages nothing. A sink which dead-letters the message, throws, or reports - /// another status stages nothing itself, so the message is dead-lettered with its payload and recorded as processed. + /// A duplicate or backpressured message stages nothing. A sink which dead-letters the message stages nothing itself, + /// so the message is dead-lettered with its payload and recorded as processed. A sink exception, or an unsupported + /// status, leaves the outcome unknown: it propagates to the caller with nothing staged, so the sender retries. /// + /// A sink reported an unsupported delivery status. private DeliveryResult? StageSinkAcceptance(InboxMessage message) { var key = message.Envelope.MessageId; foreach (var sink in _sinks) { - DeliveryResult result; - Exception? error = null; - try - { - if (!sink.TryAccept(message, out result)) continue; - if (result.Status is not (DeliveryStatus.Accepted or DeliveryStatus.Duplicate or DeliveryStatus.Backpressured or DeliveryStatus.DeadLettered)) - { - throw new InvalidOperationException($"The inbox message sink reported the unsupported delivery status '{result.Status}'."); - } - } - catch (Exception exception) - { - error = exception; - result = DeliveryResult.DeadLettered(exception.Message); - } - + if (!sink.TryAccept(message, out var result)) continue; if (result.Status is DeliveryStatus.Duplicate or DeliveryStatus.Backpressured) { Shared.Instruments.OnInboxMessageReceived(Shared.GrainType, result.Status == DeliveryStatus.Duplicate ? "duplicate" : "backpressured"); return result; } + if (result.Status is not (DeliveryStatus.Accepted or DeliveryStatus.DeadLettered)) + { + throw new InvalidOperationException($"The inbox message sink reported the unsupported delivery status '{result.Status}'."); + } + var accepted = result.Status == DeliveryStatus.Accepted; var reason = result.Message ?? "The inbox message sink dead-lettered the message."; try @@ -235,7 +228,7 @@ internal void StageLocal(InboxMessage message) } else { - LogSinkDeadLettered(Logger, error, key, GrainContext.GrainId, reason); + LogSinkDeadLettered(Logger, key, GrainContext.GrainId, reason, result.Code); StageDeadLetter(message, reason, attemptCount: 1, result.Code, new() { Reason = reason, Code = result.Code }); } } @@ -658,6 +651,6 @@ private void ValidateAttempt() [LoggerMessage(Level = LogLevel.Error, Message = "Handler threw exception for message {MessageId} to {ReceiverId}")] private static partial void LogHandlerException(ILogger logger, Exception exception, MessageId messageId, GrainId receiverId); - [LoggerMessage(Level = LogLevel.Error, Message = "Inbox message sink dead-lettered message {MessageId} to {ReceiverId}: {Reason}")] - private static partial void LogSinkDeadLettered(ILogger logger, Exception? exception, MessageId messageId, GrainId receiverId, string reason); + [LoggerMessage(Level = LogLevel.Warning, Message = "Inbox message sink dead-lettered message {MessageId} to {ReceiverId} with code {Code}: {Reason}")] + private static partial void LogSinkDeadLettered(ILogger logger, MessageId messageId, GrainId receiverId, string reason, string? code); } diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 6696586d98b..bc11d83eeb6 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -144,10 +144,14 @@ construction or activation setup, before journal recovery. message fails its `Send` instead. - `DeadLettered`: the inbox dead-letters the message with the sink's reason and optional application-defined `Code`, such as `orders/duplicate-order`, and the - sender's source receives both in a `Rejected` retirement. A sink that throws or - reports another status is treated the same way, with no code. The processed - record keeps the reason and code, so a duplicate within `DeduplicationWindow`, - such as a retry whose first answer was lost, receives the same rejection. + sender's source receives both in a `Rejected` retirement. The processed record + keeps the reason and code, so a duplicate within `DeduplicationWindow`, such as + a retry whose first answer was lost, is rejected the same way. + + A sink exception, or any other status, leaves the outcome unknown: the inbox stages + nothing and the delivery fails, so the sender's outbox retries it with backoff + until `OutboxRetryTimeout`, and a self-sent message fails its `Send`. Return + `DeadLettered` to reject a message permanently. Sink-owned messages never occupy capacity and have no inbox retry lifecycle; the sink honors their `DeliverAt`. Declined messages follow the handler path once diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 7213a478997..a0a1a6f2163 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -126,14 +126,13 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 2); } - [Theory] - [InlineData(SinkMode.Throw, null)] - [InlineData(SinkMode.DeadLetter, SourceSinkGrain.RejectionCode)] - public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithTheReceiversReasonAndCode(SinkMode mode, string? code) + [Fact] + public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithTheReceiversReasonAndCode() { + const string code = SourceSinkGrain.RejectionCode; var sender = NewGrain(); var receiver = NewGrain(); - Probe.For(receiver).Mode = mode; + Probe.For(receiver).Mode = SinkMode.DeadLetter; var messageId = NewMessageId("receiver-dead-letter"); var payload = NewPayload(); @@ -155,6 +154,66 @@ public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithT Assert.Equal(code, inboxLetter.Code); } + [Fact] + public async Task SourceMessage_WhoseSinkThrowsTransiently_IsRetriedUntilTheSinkAcceptsIt() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Probe.For(receiver).FailNextOffers(2); + var messageId = NewMessageId("sink-transient-failure"); + var payload = NewPayload(); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, outboxDeadLetters) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters)); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), state.Retired[messageId.ToString()]); + Assert.Empty(outboxDeadLetters); + var (received, inboxDeadLetters) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState().Received, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Equal(payload, received[messageId.ToString()]); + Assert.Empty(inboxDeadLetters); + Assert.Equal(3, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + } + + [Fact] + public async Task SourceMessage_WhoseSinkKeepsThrowing_DeadLettersAtTheSenderWithTheLastErrorAfterTheRetryTimeout() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Probe.For(receiver).Mode = SinkMode.Throw; + var messageId = NewMessageId("sink-persistent-failure"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + var (state, outboxDeadLetters, reasons, codes) = await OnGrainAsync(sender, context => ( + Grain(context).GetState(), + context.ActivationServices.GetRequiredService().OutboxDeadLetters, + Grain(context).GetRetirementReasons(), + Grain(context).GetRetirementCodes())); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), state.Retired[messageId.ToString()]); + var letter = Assert.Single(outboxDeadLetters); + Assert.Contains(SourceSinkGrain.SinkFailure, letter.Reason, StringComparison.Ordinal); + Assert.Equal(letter.Reason, reasons[messageId.ToString()]); + Assert.Null(letter.Code); + Assert.Empty(codes); + Assert.True(letter.AttemptCount > 1); + + var (received, inboxDeadLetters, inboxCount) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState().Received, + context.ActivationServices.GetRequiredService().InboxDeadLetters, + Inbox(context).Count)); + Assert.Empty(received); + Assert.Empty(inboxDeadLetters); + Assert.Equal(0, inboxCount); + Assert.Equal(letter.AttemptCount, Probe.For(receiver).SinkOffers); + } + [Fact] public async Task SourceMessage_CoalescesWithItsPendingReferenceAndRejectsConflicts() { @@ -468,14 +527,13 @@ public async Task SinkAcceptance_DeduplicatesWithinTheWindowAndTheSinkReportsLat Assert.Equal(0, await OnGrainAsync(receiver, context => Inbox(context).Count)); } - [Theory] - [InlineData(SinkMode.Throw, SourceSinkGrain.Rejection, null)] - [InlineData(SinkMode.DeadLetter, SourceSinkGrain.Rejection, SourceSinkGrain.RejectionCode)] - [InlineData(SinkMode.UnsupportedStatus, "The inbox message sink reported the unsupported delivery status 'HandlerNotFound'.", null)] - public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWithTheSameRejection(SinkMode mode, string reason, string? code) + [Fact] + public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWithTheSameRejection() { + const string reason = SourceSinkGrain.Rejection; + const string code = SourceSinkGrain.RejectionCode; var receiver = NewGrain(); - Probe.For(receiver).Mode = mode; + Probe.For(receiver).Mode = SinkMode.DeadLetter; var payload = NewPayload(); var message = NewInboxMessage("rejected", payload); var extension = receiver.AsReference(); @@ -520,7 +578,7 @@ public async Task SinkRejection_DeadLettersTheMessageAndAnswersItsDuplicatesWith Assert.Equal(2, Probe.For(receiver).SinkOffers); Assert.Equal(1, Probe.For(receiver).SinkAcceptances); - void AssertRejected(DeliveryResult duplicate) + static void AssertRejected(DeliveryResult duplicate) { Assert.Equal(DeliveryStatus.DeadLettered, duplicate.Status); Assert.Equal(reason, duplicate.Message); @@ -528,6 +586,36 @@ void AssertRejected(DeliveryResult duplicate) } } + [Theory] + [InlineData(SinkMode.Throw, SourceSinkGrain.SinkFailure)] + [InlineData(SinkMode.UnsupportedStatus, "The inbox message sink reported the unsupported delivery status 'HandlerNotFound'.")] + public async Task SinkFailure_FailsTheDeliveryWithoutStagingAndTheRetryIsAccepted(SinkMode mode, string error) + { + var receiver = NewGrain(); + Probe.For(receiver).Mode = mode; + var payload = NewPayload(); + var message = NewInboxMessage("sink-failure", payload); + var extension = receiver.AsReference(); + + var exception = await Assert.ThrowsAsync(async () => await extension.DeliverAsync(message, Token)); + + Assert.Equal(error, exception.Message); + var (state, count, letters) = await OnGrainAsync(receiver, context => ( + Grain(context).GetState(), + Inbox(context).Count, + context.ActivationServices.GetRequiredService().InboxDeadLetters)); + Assert.Empty(state.Received); + Assert.Equal(0, count); + Assert.Empty(letters); + Assert.Equal(0, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + + Probe.For(receiver).Mode = SinkMode.Accept; + Assert.Equal(DeliveryStatus.Accepted, (await extension.DeliverAsync(message, Token)).Status); + Assert.Equal(2, Probe.For(receiver).SinkOffers); + Assert.Equal(1, Probe.For(receiver).SinkAcceptances); + Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(receiver), payload)); + } + [Fact] public async Task SinkBackpressure_StagesNothingAndTheRetryIsAccepted() { @@ -610,12 +698,15 @@ public async Task SelfSentMessage_IsAcceptedByTheSinkInTheSendingWrite() Assert.Equal(1, Storage.CountPayloadOccurrences(Journal(grain), payload)); } - [Fact] - public async Task SelfSentMessage_FailsTheSendWhenTheSinkBackpressures() + [Theory] + [InlineData(SinkMode.Backpressure)] + [InlineData(SinkMode.Throw)] + [InlineData(SinkMode.UnsupportedStatus)] + public async Task SelfSentMessage_FailsTheSendWithoutStagingWhenTheSinkBackpressuresOrFails(SinkMode mode) { var grain = NewGrain(); _ = await grain.GetOutboxCountAsync(); - Probe.For(grain).Mode = SinkMode.Backpressure; + Probe.For(grain).Mode = mode; var writes = Writes(grain).Length; await Assert.ThrowsAsync( @@ -785,6 +876,7 @@ public sealed class Counters private int _sinkOffers; private int _sinkAcceptances; private int _handled; + private int _failuresRemaining; public volatile SinkMode Mode; public ConcurrentDictionary Faults { get; } = new(); @@ -799,6 +891,11 @@ public sealed class Counters public void OnSinkOffered() => Interlocked.Increment(ref _sinkOffers); public void OnSinkAccepted() => Interlocked.Increment(ref _sinkAcceptances); public void OnHandled() => Interlocked.Increment(ref _handled); + + /// Makes the sink throw for the next offers before applying . + public void FailNextOffers(int count) => Volatile.Write(ref _failuresRemaining, count); + + public bool TryConsumeFailure() => Interlocked.Decrement(ref _failuresRemaining) >= 0; } } @@ -820,6 +917,7 @@ public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, internal const string Subject = "source-sink/payload"; internal const string Rejection = "The sink rejected the message."; internal const string RejectionCode = "source-sink/rejected"; + internal const string SinkFailure = "The sink failed before deciding the message's outcome."; internal const string SourceFailure = "The source can't decode the stored message."; private readonly IJournaledStateManager _manager; private readonly IOutbox _outbox; @@ -965,12 +1063,13 @@ bool IInboxMessageSink.TryAccept(InboxMessage message, out DeliveryResult result if (message.Envelope.Subject != Subject) return false; _probe.OnSinkOffered(); var key = message.Envelope.MessageId.ToString(); + if (_probe.TryConsumeFailure()) throw new InvalidOperationException(SinkFailure); switch (_probe.Mode) { case SinkMode.Decline: return false; case SinkMode.Throw: - throw new InvalidOperationException(Rejection); + throw new InvalidOperationException(SinkFailure); case SinkMode.DeadLetter: result = DeliveryResult.DeadLettered(Rejection, RejectionCode); return true; From f97e4c7b19bd647b6dc0f824fe93c85a5bf8c2df Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 07:40:40 -0700 Subject: [PATCH 83/95] test(messaging): add reusable delivery fault injection seams Add DeliveryFaultInjector, a Polly strategy for DeliveryResiliencePipeline which can throw, time out, lose the receiver's response after it committed, or hold a delivery before or after the receiver call; FaultInjectionClusterFixture, which composes it with a manual Durable Jobs clock; and a cancelable pump run on MessagingClusterFixture. --- .../Support/DeliveryFaultInjector.cs | 334 ++++++++++++++++++ .../Support/FaultInjectionClusterFixture.cs | 92 +++++ .../Support/MessagingClusterFixture.cs | 9 +- 3 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 test/Orleans.Messaging.Tests/Support/DeliveryFaultInjector.cs create mode 100644 test/Orleans.Messaging.Tests/Support/FaultInjectionClusterFixture.cs diff --git a/test/Orleans.Messaging.Tests/Support/DeliveryFaultInjector.cs b/test/Orleans.Messaging.Tests/Support/DeliveryFaultInjector.cs new file mode 100644 index 00000000000..de67ffd7b57 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/DeliveryFaultInjector.cs @@ -0,0 +1,334 @@ +using System.Collections.Concurrent; +using Polly; +using Polly.Timeout; + +namespace Orleans.Messaging.Tests.Support; + +/// +/// Injects scripted faults into the outbox's call to the receiver, as a strategy of +/// . +/// +/// +/// +/// Each delivery try takes the next scripted ; once the script is exhausted, tries take a +/// fault from when one is set, or else call the receiver unchanged. Faults never wait for time to +/// pass: a timeout is reported at once, and holds wait for an explicit release or the delivery's cancellation. +/// +/// +/// The injector records the receiver's own answers, including answers which a fault hides from the sender, so tests +/// can assert what the receiver decided independently of what the sender observed. +/// +/// +public sealed class DeliveryFaultInjector : ResilienceStrategy +{ + private readonly ConcurrentQueue _script = new(); + private readonly CountSignal _executions = new(); + private readonly CountSignal _receiverCalls = new(); + private readonly CountSignal _answerCount = new(); + private readonly List _answers = []; + private Func? _repeat; + + /// Gets the number of delivery tries which reached this strategy. + public int Executions => _executions.Value; + + /// Gets the number of tries which called the receiver. + public int ReceiverCalls => _receiverCalls.Value; + + /// Gets the statuses which the receiver answered, in completion order, whether or not the sender saw them. + public IReadOnlyList Answers + { + get + { + lock (_answers) return [.. _answers]; + } + } + + /// Creates a pipeline which consists of this strategy only. + public ResiliencePipeline CreatePipeline() => AddTo(new ResiliencePipelineBuilder()).Build(); + + /// Adds this strategy to , innermost after the strategies added before it. + public ResiliencePipelineBuilder AddTo(ResiliencePipelineBuilder builder) => + builder.AddStrategy(_ => this, new Options()); + + /// Scripts for the next delivery tries, in order. + public void Enqueue(params DeliveryFault[] faults) + { + foreach (var fault in faults) _script.Enqueue(fault); + } + + /// Applies a new fault from to every try once the script is exhausted. + public void Repeat(Func create) => _repeat = create; + + /// Lets tries call the receiver unchanged once the script is exhausted. + public void StopRepeating() => _repeat = null; + + /// Waits until tries have reached this strategy. + public Task WaitForExecutionsAsync(int count) => _executions.WaitAsync(count); + + /// Waits until tries have called the receiver. + public Task WaitForReceiverCallsAsync(int count) => _receiverCalls.WaitAsync(count); + + /// Waits until the receiver has answered calls, and returns all answers so far. + public async Task> WaitForAnswersAsync(int count) + { + await _answerCount.WaitAsync(count); + return Answers; + } + + protected override ValueTask> ExecuteCore( + Func>> callback, + ResilienceContext context, + TState state) + { + _executions.Increment(); + var fault = _script.TryDequeue(out var scripted) ? scripted : _repeat?.Invoke(); + return fault is null ? CallReceiverAsync(callback, context, state, null) : PlayAsync(fault, callback, context, state); + } + + private async ValueTask> PlayAsync( + DeliveryFault fault, + Func>> callback, + ResilienceContext context, + TState state) + { + var continueOnCapturedContext = context.ContinueOnCapturedContext; + var cancellationToken = context.CancellationToken; + if (fault.Kind != DeliveryFaultKind.HoldAfterCall) fault.OnEntered(); + switch (fault.Kind) + { + case DeliveryFaultKind.Throw: + return Outcome.FromException(fault.Exception!); + case DeliveryFaultKind.TimeOut: + return Outcome.FromException(new TimeoutRejectedException("Injected delivery timeout before the receiver was called.")); + case DeliveryFaultKind.Answer: + return Outcome.FromResult((TResult)(object)fault.Result!.Value); + case DeliveryFaultKind.TimeOutInFlight: + // The abandoned call continues to the receiver; its answer is observable through ReceiverAnswer only. + _ = CallReceiverAsync(callback, context, state, fault).AsTask(); + return Outcome.FromException(new TimeoutRejectedException("Injected delivery timeout while the receiver call was in flight.")); + case DeliveryFaultKind.LoseResponse: + await CallReceiverAsync(callback, context, state, fault).ConfigureAwait(continueOnCapturedContext); + return Outcome.FromException(new TimeoutRejectedException("Injected loss of the receiver's response.")); + case DeliveryFaultKind.HoldBeforeCall: + try + { + await fault.WaitForReleaseAsync(cancellationToken).ConfigureAwait(continueOnCapturedContext); + } + catch (Exception exception) + { + return Outcome.FromException(exception); + } + + return await CallReceiverAsync(callback, context, state, fault).ConfigureAwait(continueOnCapturedContext); + case DeliveryFaultKind.HoldAfterCall: + var outcome = await CallReceiverAsync(callback, context, state, fault).ConfigureAwait(continueOnCapturedContext); + fault.OnEntered(); + try + { + await fault.WaitForReleaseAsync(cancellationToken).ConfigureAwait(continueOnCapturedContext); + } + catch (Exception exception) + { + return Outcome.FromException(exception); + } + + return outcome; + default: + throw new InvalidOperationException($"Unknown delivery fault '{fault.Kind}'."); + } + } + + private async ValueTask> CallReceiverAsync( + Func>> callback, + ResilienceContext context, + TState state, + DeliveryFault? fault) + { + var continueOnCapturedContext = context.ContinueOnCapturedContext; + _receiverCalls.Increment(); + Outcome outcome; + try + { + outcome = await callback(context, state).ConfigureAwait(continueOnCapturedContext); + } + catch (Exception exception) + { + outcome = Outcome.FromException(exception); + } + + if (outcome.Exception is { } failure) + { + fault?.OnAnswered(failure); + } + else + { + var result = (DeliveryResult)(object)outcome.Result!; + lock (_answers) _answers.Add(result.Status); + _answerCount.Increment(); + fault?.OnAnswered(result); + } + + return outcome; + } + + private sealed class Options : ResilienceStrategyOptions; + + /// A counter whose waiters complete once it reaches their target. + private sealed class CountSignal + { + private readonly List<(int Target, TaskCompletionSource Signal)> _waiters = []; + private int _value; + + public int Value + { + get + { + lock (_waiters) return _value; + } + } + + public void Increment() + { + List? ready = null; + lock (_waiters) + { + _value++; + for (var index = _waiters.Count - 1; index >= 0; index--) + { + if (_waiters[index].Target > _value) continue; + (ready ??= []).Add(_waiters[index].Signal); + _waiters.RemoveAt(index); + } + } + + ready?.ForEach(static signal => signal.TrySetResult()); + } + + public Task WaitAsync(int target) + { + lock (_waiters) + { + if (_value >= target) return Task.CompletedTask; + var signal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((target, signal)); + return signal.Task.WaitAsync(TimeSpan.FromSeconds(30)); + } + } + } +} + +/// The kinds of fault which injects. +public enum DeliveryFaultKind +{ + /// The try fails with an exception, without calling the receiver. + Throw, + + /// The try times out without calling the receiver. + TimeOut, + + /// The try returns an answer without calling the receiver. + Answer, + + /// The try calls the receiver and times out at once, while the call continues. + TimeOutInFlight, + + /// The try calls the receiver, then hides its answer behind a timeout. + LoseResponse, + + /// The try waits for a release before it calls the receiver. + HoldBeforeCall, + + /// The try calls the receiver, then waits for a release before it returns the answer. + HoldAfterCall +} + +/// +/// One single-use fault for a delivery try, and the handle through which a test observes and controls it. +/// +public sealed class DeliveryFault +{ + private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _answer = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _used; + + private DeliveryFault(DeliveryFaultKind kind, Exception? exception = null, DeliveryResult? result = null) + { + Kind = kind; + Exception = exception; + Result = result; + } + + /// Gets the kind of this fault. + public DeliveryFaultKind Kind { get; } + + internal Exception? Exception { get; } + internal DeliveryResult? Result { get; } + + /// + /// Gets the receiver's answer to the call which this fault made, or its failure. It completes only for faults which + /// call the receiver. + /// + public Task ReceiverAnswer => _answer.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + /// Fails the try with , or an by default. + public static DeliveryFault Throw(Exception? exception = null) => + new(DeliveryFaultKind.Throw, exception ?? new IOException("Injected transient delivery failure.")); + + /// Times the try out with a , without calling the receiver. + public static DeliveryFault TimeOut() => new(DeliveryFaultKind.TimeOut); + + /// Answers the try with , without calling the receiver. + public static DeliveryFault Answer(DeliveryResult result) => new(DeliveryFaultKind.Answer, result: result); + + /// + /// Calls the receiver and times the try out at once, while the call continues; reports + /// the abandoned call's answer. + /// + public static DeliveryFault TimeOutInFlight() => new(DeliveryFaultKind.TimeOutInFlight); + + /// + /// Calls the receiver and lets it decide, then fails the try with a , as when + /// the response is lost after the receiver committed. + /// + public static DeliveryFault LoseResponse() => new(DeliveryFaultKind.LoseResponse); + + /// + /// Holds the try before it calls the receiver until or , or until the delivery + /// is canceled. + /// + public static DeliveryFault HoldBeforeCall() => new(DeliveryFaultKind.HoldBeforeCall); + + /// + /// Calls the receiver, then holds its answer until or , or until the delivery + /// is canceled, which models a sender that stops after the receiver committed. + /// + public static DeliveryFault HoldAfterCall() => new(DeliveryFaultKind.HoldAfterCall); + + /// + /// Waits until a try takes this fault; for , until the receiver answered. + /// + public Task WaitUntilEnteredAsync() => _entered.Task.WaitAsync(TimeSpan.FromSeconds(30)); + + /// Releases a held try to continue. + public void Release() => _released.TrySetResult(); + + /// Releases a held try by failing it with , without the receiver's answer. + public void Fail(Exception exception) => _released.TrySetException(exception); + + internal void OnEntered() + { + if (Interlocked.Exchange(ref _used, 1) != 0) + { + throw new InvalidOperationException("A delivery fault applies to one try only."); + } + + _entered.TrySetResult(); + } + + internal Task WaitForReleaseAsync(CancellationToken cancellationToken) => _released.Task.WaitAsync(cancellationToken); + + internal void OnAnswered(DeliveryResult result) => _answer.TrySetResult(result); + + internal void OnAnswered(Exception exception) => _answer.TrySetException(exception); +} diff --git a/test/Orleans.Messaging.Tests/Support/FaultInjectionClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/FaultInjectionClusterFixture.cs new file mode 100644 index 00000000000..3bd74c4baad --- /dev/null +++ b/test/Orleans.Messaging.Tests/Support/FaultInjectionClusterFixture.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.DurableJobs; +using Orleans.Messaging.Configuration; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Support; + +/// +/// A messaging cluster whose delivery calls, journal writes, Durable Jobs scheduling, and Durable Jobs clock are all +/// under test control, for deterministic fault injection at every boundary of the delivery chain. +/// +/// +/// +/// injects faults into the outbox's call to the receiver; +/// fails or holds journal writes before or after they become durable; +/// fails or holds wake-up scheduling. drives the outbox's retry backoff and expiry and scheduled +/// delivery times, and moves only when a test advances it. +/// +/// +/// Durable Jobs fires a wake-up which is due when it is scheduled at once, but a wake-up which is due later waits in real +/// time; is long enough that such wake-ups never fire during a test. After advancing +/// , tests run the pump explicitly through . +/// +/// +/// stays zero, so a handler canceled by +/// deactivation which were wrongly counted as a failure would dead-letter at once. +/// +/// +public class FaultInjectionClusterFixture() : MessagingClusterFixture(1, receiverOnly: false) +{ + /// The first outbox retry backoff, which doubles with each further attempt. + public static readonly TimeSpan RetryInterval = TimeSpan.FromMinutes(10); + + /// How long the outbox retries a message before it dead-letters the message. + public static readonly TimeSpan OutboxRetryTimeout = TimeSpan.FromHours(1); + + /// Gets the Durable Jobs clock, which the outbox and scheduled deliveries use. + public ManualJobClock JobClock { get; } = new(DateTimeOffset.UtcNow); + + /// Gets the fault injector for the outbox's calls to receivers. + public DeliveryFaultInjector Deliveries { get; } = new(); + + /// Gets the probe for activations. + public Functional.SourceSinkProbe SourceSinkProbe { get; } = new(); + + /// Advances by , then runs one pump pass for . + public async Task AdvanceAndRunPumpAsync(IAddressable grain, TimeSpan delta) + { + JobClock.Advance(delta); + return await RunPumpAsync(grain); + } + + /// Waits until the outbox of has recorded failed attempts for . + internal Task WaitForFailedAttemptsAsync(IAddressable sender, MessageId messageId, int attempts) => + WaitUntilAsync(sender, () => + GetOutboxState(sender) is var state + && state.AttemptKeys.Contains(messageId) + && state.Attempt(messageId).AttemptCount >= attempts); + + /// Gets test access to the outbox state of the active . + internal OutboxStateTestAccess GetOutboxState(IAddressable grain) => + new(GetGrainContext(grain).ActivationServices.GetRequiredService()); + + /// + /// Deactivates as a messaging failure does, which cancels its in-flight deliveries and + /// handlers, and waits until the activation is gone. + /// + public async Task CrashAsync(IAddressable grain) + { + var context = GetGrainContext(grain); + await OnGrainAsync(grain, () => context.Deactivate( + new DeactivationReason(DeactivationReasonCode.ApplicationError, "Injected activation crash."))); + await context.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + protected override void ConfigureServices(IServiceCollection services) + { + services.AddKeyedSingleton(DurableJobTimeProviderNames.DurableJobs, JobClock); + services.AddSingleton(SourceSinkProbe); + } + + protected override void ConfigureOptions(DurableMessagingOptions options) + { + base.ConfigureOptions(options); + options.MaxCapacity = 8; + options.DeduplicationWindow = TimeSpan.FromDays(1); + options.OutboxRetryTimeout = OutboxRetryTimeout; + options.RetryInitialInterval = RetryInterval; + options.DeliveryResiliencePipeline = Deliveries.CreatePipeline(); + } +} diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index a14282c4e5d..5a026aa617b 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -194,12 +194,19 @@ await services.GetRequiredService().ScheduleJobAsync( /// job would. /// internal Task RunPumpAsync(IAddressable grain) => - RunPumpAsync(grain, TestContext.Current.CancellationToken, duplicate: false); + RunCancelablePumpAsync(grain, TestContext.Current.CancellationToken); /// Executes the pump's durable job callback directly as a job other than its current wake-up job. internal Task RunDuplicatePumpAsync(IAddressable grain) => RunPumpAsync(grain, TestContext.Current.CancellationToken, duplicate: true); + /// + /// Executes the pump's durable job callback directly with as the run's token, so a + /// test can cancel the run as Durable Jobs does. + /// + internal Task RunCancelablePumpAsync(IAddressable grain, CancellationToken runCancellation) => + RunPumpAsync(grain, runCancellation, duplicate: false); + private async Task RunPumpAsync(IAddressable grain, CancellationToken runCancellation, bool duplicate) { var pump = GetPump(grain); From 4cf6d2f0a629d42e11771b3328a309740291d1a8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 07:40:41 -0700 Subject: [PATCH 84/95] test(messaging): inject faults at each boundary of the delivery chain Assert that delivery, journal, wake-up, and lifecycle faults only delay a message, which is then processed and delivered exactly once in effect, and that only explicit rejections and expired retries are permanent. --- .../Functional/FaultInjectionTests.cs | 480 ++++++++++++++++++ 1 file changed, 480 insertions(+) create mode 100644 test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs diff --git a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs new file mode 100644 index 00000000000..341b4556a14 --- /dev/null +++ b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs @@ -0,0 +1,480 @@ +using System.Security.Cryptography; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Messaging.Tests.Support; +using Orleans.Runtime; +using Xunit; + +namespace Orleans.Messaging.Tests.Functional; + +/// +/// Injects faults at each boundary of the delivery chain and asserts that transient infrastructure faults only delay a +/// message, which is then processed and delivered exactly once in effect, while only explicit rejections and expired +/// retries are permanent. +/// +/// +/// Faults are injected through for the outbox's call to the receiver, +/// for journal writes, for wake-up +/// scheduling, and for handlers. Time moves only through +/// , and each retry runs through an explicit pump run. +/// +[Collection(MessagingClusterCollection.Name)] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Messaging")] +public sealed class FaultInjectionTests : MessagingBehaviorTestBase +{ + private const string Route = "messages/record"; + private const string HeldRoute = "messages/held"; + private static readonly TimeSpan RetryInterval = FaultInjectionClusterFixture.RetryInterval; + private static readonly TimeSpan RetryTimeout = FaultInjectionClusterFixture.OutboxRetryTimeout; + private readonly FaultInjectionClusterFixture _cluster; + + public FaultInjectionTests() : this(new FaultInjectionClusterFixture()) + { + } + + private FaultInjectionTests(FaultInjectionClusterFixture cluster) : base(cluster) => _cluster = cluster; + + private DeliveryFaultInjector Deliveries => _cluster.Deliveries; + + [Theory] + [InlineData(DeliveryFaultKind.Throw, "Injected transient delivery failure.")] + [InlineData(DeliveryFaultKind.TimeOut, "TimeoutRejectedException")] + public async Task TransientDeliveryFault_DelaysTheMessageThenDeliversItOnce(DeliveryFaultKind kind, string error) + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(kind == DeliveryFaultKind.Throw ? DeliveryFault.Throw() : DeliveryFault.TimeOut()); + var startedAt = _cluster.JobClock.GetUtcNow(); + var message = NewMessage(1, "transient"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + // The fault only schedules a retry; the receiver was never called. + var attempt = await WaitForFailedAttemptAsync(sender, id, 1); + Assert.Contains(error, attempt.LastError); + Assert.Equal(startedAt + RetryInterval, attempt.NextAttemptAt); + Assert.Equal(startedAt + RetryTimeout, attempt.ExpirationTime); + Assert.Equal(0, Deliveries.ReceiverCalls); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task LostResponse_AfterTheReceiverCommitted_IsADuplicateAtTheReceiverAndDeliveredAtTheSender() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var lost = DeliveryFault.LoseResponse(); + Deliveries.Enqueue(lost); + var message = NewMessage(2, "lost-response"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + Assert.Equal(DeliveryStatus.Accepted, (await lost.ReceiverAnswer).Status); + Assert.Contains("Injected loss of the receiver's response.", (await WaitForFailedAttemptAsync(sender, id, 1)).LastError); + await Fixture.WaitForEffectCountAsync(receiver, 1); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], Deliveries.Answers); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task LostResponse_AfterTheSinkAccepted_RetiresTheSourceMessageAsDelivered() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var lost = DeliveryFault.LoseResponse(); + Deliveries.Enqueue(lost); + var id = NewSourceMessageId("lost-response"); + var payload = RandomNumberGenerator.GetBytes(32); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + + Assert.Equal(DeliveryStatus.Accepted, (await lost.ReceiverAnswer).Status); + await WaitForFailedAttemptAsync(sender, id, 1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), (await OnSinkAsync(sender, static grain => grain.GetState())).Retired[id.ToString()]); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], Deliveries.Answers); + Assert.Equal(payload, (await OnSinkAsync(receiver, static grain => grain.GetState())).Received[id.ToString()]); + + // The receiver's deduplication answered the retry before offering it to the sink again. + var probe = _cluster.SourceSinkProbe.For(receiver); + Assert.Equal(1, probe.SinkOffers); + Assert.Equal(1, probe.SinkAcceptances); + } + + [Fact] + public async Task TimedOutDelivery_WhoseCallIsStillCommitting_IsDeduplicatedAgainstTheRetry() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + using var acceptance = Fixture.Storage.BlockWrite(JournalId.FromGrainId(receiver.GetGrainId())); + var timeout = DeliveryFault.TimeOutInFlight(); + Deliveries.Enqueue(timeout); + var message = NewMessage(3, "in-flight"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + // The sender gave up on the call while the receiver's acceptance write is still in progress, then retries. + await acceptance.WaitUntilEnteredAsync(); + await WaitForFailedAttemptAsync(sender, id, 1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await Deliveries.WaitForReceiverCallsAsync(2); + acceptance.Release(); + + var answers = await Deliveries.WaitForAnswersAsync(2); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], answers.Order()); + Assert.Equal(DeliveryStatus.Accepted, (await timeout.ReceiverAnswer).Status); + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + } + + [Fact] + public async Task PersistentDeliveryFault_DeadLettersAtTheSenderWithTheLastErrorAfterTheRetryTimeout() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + Deliveries.Repeat(static () => DeliveryFault.Throw(new IOException("Injected outage of the receiver's silo."))); + var startedAt = _cluster.JobClock.GetUtcNow(); + var id = NewSourceMessageId("persistent-fault"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: true); + + await WaitForFailedAttemptAsync(sender, id, 1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + var second = await WaitForFailedAttemptAsync(sender, id, 2); + Assert.Equal(startedAt + RetryTimeout, second.ExpirationTime); + Assert.Equal(startedAt + RetryInterval + 2 * RetryInterval, second.NextAttemptAt); + + // Once the retry timeout passes, the next due attempt dead-letters the message without another try. + await _cluster.AdvanceAndRunPumpAsync(sender, RetryTimeout); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + var (retired, reasons, deadLetter) = await Fixture.OnGrainAsync(sender, () => ( + Sink(sender).GetState().Retired, + Sink(sender).GetRetirementReasons(), + _cluster.GetOutboxState(sender).DeadLetter(id))); + Assert.Equal(nameof(OutboxMessageRetirementStatus.DeadLettered), retired[id.ToString()]); + Assert.Contains("Injected outage of the receiver's silo.", reasons[id.ToString()]); + Assert.Equal(reasons[id.ToString()], deadLetter.Reason); + Assert.Equal(2, deadLetter.AttemptCount); + Assert.Equal(2, Deliveries.Executions); + Assert.Equal(0, Deliveries.ReceiverCalls); + Assert.Equal(0, _cluster.SourceSinkProbe.For(receiver).SinkOffers); + } + + [Fact] + public async Task SinkBackpressure_ThroughTheOutbox_RetriesBeyondTheRetryTimeoutUntilTheSinkAccepts() + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var probe = _cluster.SourceSinkProbe.For(receiver); + probe.Mode = SinkMode.Backpressure; + var id = NewSourceMessageId("backpressure"); + var payload = RandomNumberGenerator.GetBytes(32); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, payload, write: true); + + var first = await WaitForFailedAttemptAsync(sender, id, 1); + Assert.Null(first.ExpirationTime); + Assert.Equal("The receiver is backpressured.", first.LastError); + + // Backpressure proves the receiver took no earlier attempt, so it never starts the retry timeout. + await _cluster.AdvanceAndRunPumpAsync(sender, 2 * RetryTimeout); + var second = await WaitForFailedAttemptAsync(sender, id, 2); + Assert.Null(second.ExpirationTime); + Assert.Empty(await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).DeadLetterKeys.ToArray())); + + probe.Mode = SinkMode.Accept; + await _cluster.AdvanceAndRunPumpAsync(sender, 2 * RetryInterval); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Delivered), (await OnSinkAsync(sender, static grain => grain.GetState())).Retired[id.ToString()]); + Assert.Equal([DeliveryStatus.Backpressured, DeliveryStatus.Backpressured, DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal(3, probe.SinkOffers); + Assert.Equal(1, probe.SinkAcceptances); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReceiverAcceptanceWriteFailure_IsRetriedBySenderAndProcessedOnce(bool ambiguous) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (ambiguous) + { + Fixture.Storage.FailAfterWrite(journal); + } + else + { + Fixture.Storage.FailWrite(journal); + } + + var message = NewMessage(4, ambiguous ? "ambiguous-acceptance" : "failed-acceptance"); + + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + + // The receiver failed the call and deactivated; the sender holds the message for a retry. + Assert.Contains("Injected", (await WaitForFailedAttemptAsync(sender, id, 1)).LastError); + Assert.Empty(Deliveries.Answers); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await WaitForDrainedOutboxAsync(sender); + // An acceptance which became durable despite the failure is recovered, so the retry is a duplicate. + Assert.Equal([ambiguous ? DeliveryStatus.Duplicate : DeliveryStatus.Accepted], Deliveries.Answers); + var processed = await AssertProcessedOnceAsync(receiver, message); + Assert.NotEqual(before.ActivationId, processed.ActivationId); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SenderRetryStateWriteFailure_RecoversAndDeliversOnce(bool ambiguous) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var hold = DeliveryFault.HoldBeforeCall(); + Deliveries.Enqueue(hold); + var startedAt = _cluster.JobClock.GetUtcNow(); + var message = NewMessage(5, ambiguous ? "ambiguous-retry-state" : "failed-retry-state"); + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + await hold.WaitUntilEnteredAsync(); + ArmSenderWriteFailure(sender, ambiguous); + var previous = Fixture.GetGrainContext(sender); + + // The delivery fails, and so does the write which records the failed attempt. + hold.Fail(new IOException("Injected delivery failure.")); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await sender.GetSnapshotAsync(); + + if (ambiguous) + { + // The recorded attempt became durable, so the recovered message waits for its backoff. + var recovered = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(1, recovered.AttemptCount); + Assert.Equal(startedAt + RetryInterval, recovered.NextAttemptAt); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + } + + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal(1, Deliveries.ReceiverCalls); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SenderRemovalWriteFailure_AfterTheReceiverAccepted_DeliversOnceInEffect(bool ambiguous) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var hold = DeliveryFault.HoldAfterCall(); + Deliveries.Enqueue(hold); + var message = NewMessage(6, ambiguous ? "ambiguous-removal" : "failed-removal"); + _ = await sender.SendAsync(receiver.GetGrainId(), Route, message); + await hold.WaitUntilEnteredAsync(); + Assert.Equal(DeliveryStatus.Accepted, (await hold.ReceiverAnswer).Status); + ArmSenderWriteFailure(sender, ambiguous); + var previous = Fixture.GetGrainContext(sender); + + hold.Release(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await sender.GetSnapshotAsync(); + + await WaitForDrainedOutboxAsync(sender); + // A removal which never became durable is redelivered, and the receiver reports the duplicate. + DeliveryStatus[] expected = ambiguous ? [DeliveryStatus.Accepted] : [DeliveryStatus.Accepted, DeliveryStatus.Duplicate]; + Assert.Equal(expected, Deliveries.Answers); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task SenderCrash_AfterTheReceiverCommitted_RedeliversAsADuplicate() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var hold = DeliveryFault.HoldAfterCall(); + Deliveries.Enqueue(hold); + var message = NewMessage(7, "sender-crash"); + _ = await sender.SendAsync(receiver.GetGrainId(), Route, message); + await hold.WaitUntilEnteredAsync(); + Assert.Equal(DeliveryStatus.Accepted, (await hold.ReceiverAnswer).Status); + + // Deactivation cancels the in-flight delivery, whose outcome is never applied or counted as an attempt. + await _cluster.CrashAsync(sender); + _ = await sender.GetSnapshotAsync(); + + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted, DeliveryStatus.Duplicate], Deliveries.Answers); + await AssertProcessedOnceAsync(receiver, message); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + } + + [Fact] + public async Task ReceiverCrash_MidBatch_RecoveryProcessesEachMessageOnce() + { + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var deliverAt = _cluster.JobClock.GetUtcNow() + RetryTimeout; + var first = NewMessage(8, "first"); + var held = NewMessage(9, "held"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, first, "messages/first"), deliverAt))).Status); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, Schedule(CreateEnvelope(receiver, held, HeldRoute), deliverAt))).Status); + var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute); + + // Both messages become due in one batch; the first is handled and the second is in its handler at the crash. + _cluster.JobClock.Advance(RetryTimeout); + var run = Fixture.RunPumpAsync(receiver); + await WaitForBarrierAsync(receiver, barrier); + Assert.Equal(1, Assert.Single((await receiver.GetSnapshotAsync()).Effects).Count); + await _cluster.CrashAsync(receiver); + barrier.Dispose(); + _ = run.ContinueWith(static task => task.Exception, TaskScheduler.Default); + + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + static snapshot => snapshot.Effects.Count == 2 && snapshot.InboxCount == 0); + Assert.All(recovered.Effects, static effect => Assert.Equal(1, effect.Count)); + Assert.Equal([first.LogicalId, held.LogicalId], recovered.Effects.Select(static effect => effect.LogicalId)); + Assert.Empty(recovered.InboxDeadLetters); + } + + [Fact] + public async Task CanceledPumpRun_WhoseWakeupRepairFails_RecoversAfterReactivation() + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var barrier = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), HeldRoute); + var message = NewMessage(10, "canceled-run"); + Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(receiver, CreateEnvelope(receiver, message, HeldRoute))).Status); + await WaitForBarrierAsync(receiver, barrier); + var previous = Fixture.GetGrainContext(receiver); + + // Durable Jobs cancels a run while the handler is still working, and re-establishing the wake-up fails. + Fixture.JobManagerProbe.FailNext(MessagingPump.JobName); + using var cancellation = new CancellationTokenSource(); + var run = Fixture.RunCancelablePumpAsync(receiver, cancellation.Token); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => run); + + // Without a wake-up the activation can't guarantee progress, so it deactivates and recovery takes over. + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + barrier.Dispose(); + _ = await receiver.GetSnapshotAsync(); + + var processed = await AssertProcessedOnceAsync(receiver, message); + Assert.NotEqual(before.ActivationId, processed.ActivationId); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ScheduledDeliveryWakeupSchedulingFailure_IsRetriedBySenderAndProcessedAtDeliverAt(bool scheduledBeforeFailure) + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await receiver.GetSnapshotAsync(); + var receiverContext = Fixture.GetGrainContext(receiver); + var hold = DeliveryFault.HoldBeforeCall(); + Deliveries.Enqueue(hold); + var deliverAt = _cluster.JobClock.GetUtcNow() + RetryTimeout + RetryTimeout; + var message = NewMessage(11, "scheduled"); + var outgoing = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message); + outgoing = outgoing with { Envelope = outgoing.Envelope with { DeliverAt = deliverAt } }; + await sender.StageOutputAsync(outgoing); + await sender.RetryWriteStateAsync(); + await hold.WaitUntilEnteredAsync(); + + // The receiver's acceptance write schedules its wake-up for the message, and the scheduling fails. + if (scheduledBeforeFailure) + { + Fixture.JobManagerProbe.FailAfterNext(MessagingPump.JobName); + } + else + { + Fixture.JobManagerProbe.FailNext(MessagingPump.JobName); + } + + hold.Release(); + Assert.Contains("Injected", (await WaitForFailedAttemptAsync(sender, outgoing.Envelope.MessageId, 1)).LastError); + await receiverContext.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + Assert.Empty(Deliveries.Answers); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + var pending = await Fixture.WaitForInboxCountAsync(receiver, 1); + Assert.Empty(pending.Effects); + + await _cluster.AdvanceAndRunPumpAsync(receiver, RetryTimeout + RetryTimeout); + await AssertProcessedOnceAsync(receiver, message); + } + + private void ArmSenderWriteFailure(IMessagingTestGrain sender, bool ambiguous) + { + var journal = JournalId.FromGrainId(sender.GetGrainId()); + if (ambiguous) + { + Fixture.Storage.FailAfterWrite(journal); + } + else + { + Fixture.Storage.FailWrite(journal); + } + } + + private async Task WaitForFailedAttemptAsync(IAddressable sender, MessageId id, int attempts) + { + await _cluster.WaitForFailedAttemptsAsync(sender, id, attempts); + return await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + } + + private Task WaitForDrainedOutboxAsync(IMessagingTestGrain sender) => + Fixture.WaitUntilAsync(sender, () => Fixture.GetStagedOutput(sender).Length == 0); + + private async Task AssertProcessedOnceAsync(IMessagingTestGrain receiver, TestMessage message) + { + var processed = await Fixture.SnapshotProbe.WaitAsync( + receiver.GetGrainId(), + snapshot => snapshot.Effects.Any(effect => effect.LogicalId == message.LogicalId) && snapshot.InboxCount == 0); + var current = await receiver.GetSnapshotAsync(); + Assert.Equal(1, Assert.Single(current.Effects).Count); + Assert.Equal(message.LogicalId, current.Effects[0].LogicalId); + Assert.Equal(0, current.InboxCount); + Assert.Empty(current.InboxDeadLetters); + return processed; + } + + private ISourceSinkGrain NewSinkGrain() => Fixture.Client.GetGrain(Guid.NewGuid()); + + private SourceSinkGrain Sink(IAddressable grain) => Assert.IsType(Fixture.GetGrainContext(grain).GrainInstance); + + private Task OnSinkAsync(IAddressable grain, Func func) => + Fixture.OnGrainAsync(grain, () => func(Sink(grain))); + + private static MessageId NewSourceMessageId(string name) => MessageId.Create("fault-injection", name, Guid.NewGuid().ToString("N")); + + private static InboxMessage Schedule(InboxMessage message, DateTimeOffset deliverAt) => + new() { Envelope = message.Envelope with { DeliverAt = deliverAt } }; +} From 8dc205e89cd35540cafe15a2c105b5665e04ca6e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 09:08:21 -0700 Subject: [PATCH 85/95] feat(messaging): report inbox and outbox transitions through diagnostic events MessagingEvents, the Orleans.Messaging diagnostic listener, reports transitions synchronously on the activation which makes them, with its IGrainContext: - InboxMessageReceiving: a delivery attempt reached the inbox and isn't a duplicate, before the inbox stages anything for it. An observer's exception fails the attempt like a sink exception, so the sender retries; tests use it to inject delivery faults. - OutboxRetryScheduled: a failed delivery attempt was rescheduled, with its attempt count, error, and next attempt time. - OutboxMessageRetired: a message retired as delivered, rejected, or dead-lettered. Tests wait for these transitions instead of polling. --- .../Diagnostics/MessagingEvents.cs | 188 ++++++++++++++++++ src/Orleans.Messaging/Inbox.cs | 3 + src/Orleans.Messaging/Outbox.cs | 14 +- src/Orleans.Messaging/README.md | 7 + .../Orleans.Messaging/Orleans.Messaging.cs | 38 ++++ .../Functional/MessageSourceAndSinkTests.cs | 79 ++++++++ 6 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 src/Orleans.Messaging/Diagnostics/MessagingEvents.cs diff --git a/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs b/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs new file mode 100644 index 00000000000..8071de2d730 --- /dev/null +++ b/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Orleans.Runtime; + +namespace Orleans.Messaging.Diagnostics; + +/// +/// Provides the diagnostic listener and event payload types for durable messaging events. +/// +/// +/// +/// These types are public but may change between minor versions. They are intended for advanced scenarios such as +/// simulation testing and diagnostics. +/// +/// +/// Each event is written synchronously on the activation which raises it, at the transition which it describes, so an +/// observer sees the grain's state at that transition. An exception thrown by an observer propagates to the code which +/// raised the event; documents how a delivery treats one. +/// +/// +public static class MessagingEvents +{ + /// The name of the diagnostic listener for durable messaging events. + public const string ListenerName = "Orleans.Messaging"; + + private static readonly DiagnosticListener Listener = new(ListenerName); + + /// Gets an observable sequence of all durable messaging events. + public static IObservable AllEvents { get; } = new Observable(); + + /// The base class used for durable messaging diagnostic events. + /// The activation whose inbox or outbox raised the event. + public abstract class MessagingEvent(IGrainContext grainContext) + { + /// The activation whose inbox or outbox raised the event. + public readonly IGrainContext GrainContext = grainContext; + } + + /// + /// Event payload for a delivery attempt which reached a grain's inbox and isn't a duplicate, before the inbox stages + /// anything for the message. + /// + /// The receiving activation. + /// The delivered message. + /// + /// An exception thrown by an observer fails the delivery attempt like an inbox message sink exception: the inbox + /// stages nothing, and the sender's outbox retries the message, which tests use to inject delivery faults. + /// + public sealed class InboxMessageReceiving(IGrainContext grainContext, InboxMessage message) : MessagingEvent(grainContext) + { + /// The delivered message. + public readonly InboxMessage Message = message; + } + + /// + /// Event payload for a failed delivery attempt whose retry the outbox staged. + /// + /// The sending activation. + /// The message's ID. + /// The receiving grain. + /// The number of failed delivery attempts so far. + /// The time, on the Durable Jobs clock, from which the outbox attempts the delivery again. + /// The attempt's failure. + public sealed class OutboxRetryScheduled( + IGrainContext grainContext, + MessageId messageId, + GrainId receiverId, + int attemptCount, + DateTimeOffset nextAttemptAt, + string error) : MessagingEvent(grainContext) + { + /// The message's ID. + public readonly MessageId MessageId = messageId; + + /// The receiving grain. + public readonly GrainId ReceiverId = receiverId; + + /// The number of failed delivery attempts so far. + public readonly int AttemptCount = attemptCount; + + /// The time, on the Durable Jobs clock, from which the outbox attempts the delivery again. + public readonly DateTimeOffset NextAttemptAt = nextAttemptAt; + + /// The attempt's failure. + public readonly string Error = error; + } + + /// + /// Event payload for an outbox message whose retirement the outbox staged: the message was delivered, its receiver + /// rejected it, or the outbox dead-lettered it. + /// + /// The sending activation. + /// The message's ID. + /// How the message retired. + /// + /// The retirement commits with the sending grain's next journal write. A reference message's source is notified + /// first, so its changes for the retirement are staged when observers run. + /// + public sealed class OutboxMessageRetired( + IGrainContext grainContext, + MessageId messageId, + OutboxMessageRetirement retirement) : MessagingEvent(grainContext) + { + /// The message's ID. + public readonly MessageId MessageId = messageId; + + /// How the message retired. + public readonly OutboxMessageRetirement Retirement = retirement; + } + + internal static void EmitInboxMessageReceiving(IGrainContext grainContext, InboxMessage message) + { + if (!Listener.IsEnabled(nameof(InboxMessageReceiving))) + { + return; + } + + Emit(grainContext, message); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void Emit(IGrainContext grainContext, InboxMessage message) => + Listener.Write(nameof(InboxMessageReceiving), new InboxMessageReceiving(grainContext, message)); + } + + internal static void EmitOutboxRetryScheduled( + IGrainContext grainContext, + MessageId messageId, + GrainId receiverId, + int attemptCount, + DateTimeOffset nextAttemptAt, + string error) + { + if (!Listener.IsEnabled(nameof(OutboxRetryScheduled))) + { + return; + } + + Emit(grainContext, messageId, receiverId, attemptCount, nextAttemptAt, error); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void Emit( + IGrainContext grainContext, + MessageId messageId, + GrainId receiverId, + int attemptCount, + DateTimeOffset nextAttemptAt, + string error) => + Listener.Write( + nameof(OutboxRetryScheduled), + new OutboxRetryScheduled(grainContext, messageId, receiverId, attemptCount, nextAttemptAt, error)); + } + + internal static void EmitOutboxMessageRetired(IGrainContext grainContext, MessageId messageId, OutboxMessageRetirement retirement) + { + if (!Listener.IsEnabled(nameof(OutboxMessageRetired))) + { + return; + } + + Emit(grainContext, messageId, retirement); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void Emit(IGrainContext grainContext, MessageId messageId, OutboxMessageRetirement retirement) => + Listener.Write(nameof(OutboxMessageRetired), new OutboxMessageRetired(grainContext, messageId, retirement)); + } + + private sealed class Observable : IObservable + { + public IDisposable Subscribe(IObserver observer) => Listener.Subscribe(new Observer(observer)); + + private sealed class Observer(IObserver observer) : IObserver> + { + public void OnCompleted() => observer.OnCompleted(); + + public void OnError(Exception error) => observer.OnError(error); + + public void OnNext(KeyValuePair value) + { + if (value.Value is MessagingEvent evt) + { + observer.OnNext(evt); + } + } + } + } +} diff --git a/src/Orleans.Messaging/Inbox.cs b/src/Orleans.Messaging/Inbox.cs index c60feeb13f7..f24f788f29c 100644 --- a/src/Orleans.Messaging/Inbox.cs +++ b/src/Orleans.Messaging/Inbox.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using Orleans.Journaling; using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; using Orleans.Runtime; namespace Orleans.Messaging; @@ -98,6 +99,8 @@ public async ValueTask DeliverAsync(InboxMessage message, Cancel : DeliveryResult.Duplicate(); } + MessagingEvents.EmitInboxMessageReceiving(GrainContext, message); + // Sink-owned messages never occupy inbox capacity and need no handler. A sink's duplicate can still be staged, // so every outcome except backpressure is reported once the write is durable. if (StageSinkAcceptance(message) is { } sinkResult) diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index d033f2e5174..489dcf8f837 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; using Orleans.Journaling; using Orleans.Runtime; using Polly; @@ -185,9 +186,10 @@ private bool TryGetSourceMessage(IOutboxMessageSource source, MessageId messageI } /// Notifies the source of a retired reference entry, so its release commits with the entry's removal. - private static void Retire(OutboxEntry entry, OutboxMessageRetirement retirement) + private void Retire(OutboxEntry entry, OutboxMessageRetirement retirement) { if (entry.Reference is not null) entry.Source!.OnRetired(entry.MessageId, retirement); + MessagingEvents.EmitOutboxMessageRetired(_pump.GrainContext, entry.MessageId, retirement); } /// @@ -447,6 +449,16 @@ private async ValueTask ApplyDeliveriesAsync(DeliveryOutcome[] outcomes) var (retry, deadLetter) = PrepareRetry(outcome, ref deadLetters); Emit(new() { Kind = OutboxCommandKind.Delivery, MessageId = id, State = retry, DeadLetter = deadLetter }); if (deadLetter is not null) Retire(current, OutboxMessageRetirement.DeadLettered(deadLetter.Reason)); + else + { + MessagingEvents.EmitOutboxRetryScheduled( + _pump.GrainContext, + id, + outcome.Message.ReceiverId, + retry!.AttemptCount, + retry.NextAttemptAt!.Value, + retry.LastError!); + } } RecordDeliveryMetrics(outcome); applied++; diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index bc11d83eeb6..2d787f43f16 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -222,3 +222,10 @@ to outbox delivery or depth. Reference sends count as sent; a reference entry wh source fails to provide its message counts as delivered `source_error`. A message an inbox sink accepts counts as received `accepted` and processed `success`; one which the sink dead-letters counts as received and processed `dead_lettered`. + +`MessagingEvents`, the `Orleans.Messaging` diagnostic listener, reports each transition +synchronously on the activation which makes it: `InboxMessageReceiving` before the inbox +stages a delivered message, `OutboxRetryScheduled` when a failed delivery is rescheduled, +and `OutboxMessageRetired` when a message retires. Each event carries the activation's +`IGrainContext`. Tests use the events as barriers, and inject a delivery fault by throwing +from an `InboxMessageReceiving` observer, which fails the attempt like a sink exception. diff --git a/src/api/Orleans.Messaging/Orleans.Messaging.cs b/src/api/Orleans.Messaging/Orleans.Messaging.cs index 1068f79650c..f8538f59101 100644 --- a/src/api/Orleans.Messaging/Orleans.Messaging.cs +++ b/src/api/Orleans.Messaging/Orleans.Messaging.cs @@ -404,6 +404,44 @@ public void Validate() { } } } +namespace Orleans.Messaging.Diagnostics +{ + public static partial class MessagingEvents + { + public const string ListenerName = "Orleans.Messaging"; + public static System.IObservable AllEvents { get { throw null; } } + + public sealed partial class InboxMessageReceiving : MessagingEvent + { + public readonly InboxMessage Message; + public InboxMessageReceiving(Runtime.IGrainContext grainContext, InboxMessage message) : base(default!) { } + } + + public abstract partial class MessagingEvent + { + public readonly Runtime.IGrainContext GrainContext; + protected MessagingEvent(Runtime.IGrainContext grainContext) { } + } + + public sealed partial class OutboxMessageRetired : MessagingEvent + { + public readonly MessageId MessageId; + public readonly OutboxMessageRetirement Retirement; + public OutboxMessageRetired(Runtime.IGrainContext grainContext, MessageId messageId, OutboxMessageRetirement retirement) : base(default!) { } + } + + public sealed partial class OutboxRetryScheduled : MessagingEvent + { + public readonly int AttemptCount; + public readonly string Error; + public readonly MessageId MessageId; + public readonly System.DateTimeOffset NextAttemptAt; + public readonly Runtime.GrainId ReceiverId; + public OutboxRetryScheduled(Runtime.IGrainContext grainContext, MessageId messageId, Runtime.GrainId receiverId, int attemptCount, System.DateTimeOffset nextAttemptAt, string error) : base(default!) { } + } + } +} + namespace OrleansCodeGen.Orleans.Messaging { [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index a0a1a6f2163..83401bf85f0 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -9,6 +9,7 @@ using Orleans.Hosting; using Orleans.Journaling; using Orleans.Journaling.Json; +using Orleans.Messaging.Diagnostics; using Orleans.Messaging.Tests.Support; using Orleans.Runtime; using Orleans.TestingHost; @@ -214,6 +215,28 @@ public async Task SourceMessage_WhoseSinkKeepsThrowing_DeadLettersAtTheSenderWit Assert.Equal(letter.AttemptCount, Probe.For(receiver).SinkOffers); } + [Fact] + public async Task DiagnosticEvents_InjectADeliveryFaultAndReportTheRetryAndTheRetirement() + { + var sender = NewGrain(); + var receiver = NewGrain(); + var messageId = NewMessageId("diagnostic-events"); + using var events = new DeliveryEventObserver(sender.GetGrainId(), receiver.GetGrainId(), messageId, faults: 1); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: true); + + // The observer's exception failed the first attempt before the inbox offered the message to its sink, so the + // outbox retried the delivery, and the retry delivered it. + var retired = await events.Retired.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(sender.GetGrainId(), retired.GrainContext.GrainId); + Assert.Equal(OutboxMessageRetirementStatus.Delivered, retired.Retirement.Status); + var retry = Assert.Single(events.Retries); + Assert.Equal((sender.GetGrainId(), receiver.GetGrainId(), 1), (retry.GrainContext.GrainId, retry.ReceiverId, retry.AttemptCount)); + Assert.Contains(DeliveryEventObserver.Fault, retry.Error, StringComparison.Ordinal); + Assert.Equal(2, events.Receivings); + Assert.Equal((1, 1), (Probe.For(receiver).SinkOffers, Probe.For(receiver).SinkAcceptances)); + } + [Fact] public async Task SourceMessage_CoalescesWithItsPendingReferenceAndRejectsConflicts() { @@ -777,6 +800,62 @@ private async Task WaitUntilAsync(IAddressable grain, Func } } +/// +/// Observes one message's messaging events, and fails its first deliveries by throwing from +/// . +/// +internal sealed class DeliveryEventObserver : IObserver, IDisposable +{ + public const string Fault = "The test injected a delivery fault."; + + private readonly GrainId _sender; + private readonly GrainId _receiver; + private readonly MessageId _messageId; + private readonly IDisposable _subscription; + private readonly ConcurrentQueue _retries = new(); + private readonly TaskCompletionSource _retired = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _faults; + private int _receivings; + + public DeliveryEventObserver(GrainId sender, GrainId receiver, MessageId messageId, int faults) + { + (_sender, _receiver, _messageId, _faults) = (sender, receiver, messageId, faults); + _subscription = MessagingEvents.AllEvents.Subscribe(this); + } + + public Task Retired => _retired.Task; + public IReadOnlyCollection Retries => _retries; + public int Receivings => Volatile.Read(ref _receivings); + + public void OnNext(MessagingEvents.MessagingEvent value) + { + switch (value) + { + case MessagingEvents.InboxMessageReceiving receiving + when receiving.GrainContext.GrainId == _receiver && receiving.Message.Envelope.MessageId == _messageId: + Interlocked.Increment(ref _receivings); + if (Interlocked.Decrement(ref _faults) >= 0) + { + throw new InvalidOperationException(Fault); + } + + break; + case MessagingEvents.OutboxRetryScheduled retry when retry.GrainContext.GrainId == _sender && retry.MessageId == _messageId: + _retries.Enqueue(retry); + break; + case MessagingEvents.OutboxMessageRetired retired when retired.GrainContext.GrainId == _sender && retired.MessageId == _messageId: + _retired.TrySetResult(retired); + break; + } + } + + public void OnCompleted() { } + + public void OnError(Exception error) { } + + public void Dispose() => _subscription.Dispose(); +} + [TestSuite("BVT")] [TestProvider("None")] [TestArea("Messaging")] From 89886c98be1db72c350bfa93acbfef88898376c8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 15 Oct 2026 12:18:07 -0700 Subject: [PATCH 86/95] fix(messaging): make diagnostic event emission trim-safe DiagnosticSource.Write requires unreferenced code. MessagingEvents writes every event through one helper whose generic parameter preserves the payload's public fields and properties, so the event payloads survive trimming and the audit build reports no IL warnings for Orleans.Messaging. --- .../Diagnostics/MessagingEvents.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs b/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs index 8071de2d730..3be0833a180 100644 --- a/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs +++ b/src/Orleans.Messaging/Diagnostics/MessagingEvents.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Orleans.Runtime; @@ -121,7 +122,7 @@ internal static void EmitInboxMessageReceiving(IGrainContext grainContext, Inbox [MethodImpl(MethodImplOptions.NoInlining)] static void Emit(IGrainContext grainContext, InboxMessage message) => - Listener.Write(nameof(InboxMessageReceiving), new InboxMessageReceiving(grainContext, message)); + Write(nameof(InboxMessageReceiving), new InboxMessageReceiving(grainContext, message)); } internal static void EmitOutboxRetryScheduled( @@ -147,7 +148,7 @@ static void Emit( int attemptCount, DateTimeOffset nextAttemptAt, string error) => - Listener.Write( + Write( nameof(OutboxRetryScheduled), new OutboxRetryScheduled(grainContext, messageId, receiverId, attemptCount, nextAttemptAt, error)); } @@ -163,9 +164,16 @@ internal static void EmitOutboxMessageRetired(IGrainContext grainContext, Messag [MethodImpl(MethodImplOptions.NoInlining)] static void Emit(IGrainContext grainContext, MessageId messageId, OutboxMessageRetirement retirement) => - Listener.Write(nameof(OutboxMessageRetired), new OutboxMessageRetired(grainContext, messageId, retirement)); + Write(nameof(OutboxMessageRetired), new OutboxMessageRetired(grainContext, messageId, retirement)); } + [UnconditionalSuppressMessage( + "Trimming", + "IL2026", + Justification = "Event payloads are sealed types whose public fields and properties the generic annotation preserves, and subscribers consume the typed payload rather than reflecting over the members of the values it carries.")] + private static void Write<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.PublicProperties)] TEvent>(string name, TEvent payload) + where TEvent : MessagingEvent => Listener.Write(name, payload); + private sealed class Observable : IObservable { public IDisposable Subscribe(IObserver observer) => Listener.Subscribe(new Observer(observer)); From 20970a650d18bae43d939e8f9058db260fa1e041 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 23:10:38 -0700 Subject: [PATCH 87/95] fix(messaging): resume pending messages when a handler registers instead of polling A pending message without a registered handler kept the pump's durable job rescheduling itself every second. Messages now count toward the wake-up only when a handler is registered, and registering a handler while messages are pending signals the pump and ensures a wake-up. --- .../Functional/SelfAddressedMessageTests.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs index 98e1671b0e5..7b3de9f9f1d 100644 --- a/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/SelfAddressedMessageTests.cs @@ -169,15 +169,13 @@ public async Task SelfSend_IsAdmittedBeyondCapacityAndWithoutHandlers() // Unscheduled self-sent messages occupy capacity for deliveries from other grains. var external = CreateEnvelope(grain, NewMessage(20, "external")); Assert.Equal(DeliveryStatus.Backpressured, (await DeliverAsync(grain, external)).Status); - Assert.NotEqual(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(grain)).Status); var waiting = await grain.GetSnapshotAsync(); Assert.Equal(3, waiting.InboxCount); Assert.Empty(waiting.Effects); Assert.Empty(waiting.InboxDeadLetters); await grain.ConfigureHandlerAsync(true); - await ReactivateAsync(grain); - _ = await Fixture.RunPumpAsync(grain); var completed = await Fixture.WaitForEffectCountAsync(grain, 3); Assert.Equal(0, completed.InboxCount); Assert.Equal(DeliveryStatus.Accepted, (await DeliverAsync(grain, external)).Status); From b7357d644210c4d8e88213cd6f9f7c4dee8f6906 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 23:11:04 -0700 Subject: [PATCH 88/95] fix(messaging): schedule an earlier wake-up for work due before the current one The pump tracks the job which wakes the grain next and its due time. Work staged with an earlier due time, including a DeliverAt time or retry, schedules an earlier wake-up; the job which a newer, earlier wake-up supersedes completes when it runs. Staged due times are cleared when a write captures them, so repeated and retried preparations re-ensure the wake-up. ManualJobClock timers fire at least every 100 ms so Durable Jobs starts shards and runs wake-ups promptly after a test advances the clock. --- src/Orleans.Messaging/Outbox.State.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Messaging/Outbox.State.cs b/src/Orleans.Messaging/Outbox.State.cs index 41fee1c43d5..b98c4a14ad0 100644 --- a/src/Orleans.Messaging/Outbox.State.cs +++ b/src/Orleans.Messaging/Outbox.State.cs @@ -16,7 +16,9 @@ internal sealed partial class Outbox private long _durableSequence; ValueTask IStateMachine.PrepareWriteAsync(CancellationToken cancellationToken) => - _lastSequence != _capturedSequence ? _pump.EnsureWakeupAsync(cancellationToken) : ValueTask.CompletedTask; + _lastSequence != _capturedSequence + ? _pump.EnsureWakeupAsync(Shared.JobTimeProvider.GetUtcNow(), cancellationToken) + : ValueTask.CompletedTask; private void OnWriteCaptured() => _capturedSequence = _lastSequence; From d2c0993f3143768b644b34f394e8621b54452b1c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 23:11:21 -0700 Subject: [PATCH 89/95] feat(messaging)!: make inbox handler selection read-only IInboxHandler.CanHandle now receives the InboxMessage instead of the mutable IInboxHandlerContext. A handler can only stage a terminal outcome from HandleAsync, after the inbox selects it. BREAKING CHANGE: IInboxHandler.CanHandle(IInboxHandlerContext) is replaced by CanHandle(InboxMessage). Orleans.Messaging is alpha and has no package validation baseline. --- .../Functional/JsonJournalMessagingTests.cs | 2 +- test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs | 2 +- test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs | 2 +- test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs index a6c6ff7a876..bc55ffae1ff 100644 --- a/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/JsonJournalMessagingTests.cs @@ -159,7 +159,7 @@ public CustomHandlerGrain(IInbox inbox, IGrainContext context, JsonMessagingProb private sealed class Handler(int id, bool canHandle, IGrainContext grain, JsonMessagingProbe probe) : IInboxHandler { - public bool CanHandle(IInboxHandlerContext context) => canHandle && context.Message.Envelope.Subject == "custom"; + public bool CanHandle(InboxMessage message) => canHandle && message.Envelope.Subject == "custom"; public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs index b407d72738e..4e883bd7c30 100644 --- a/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs +++ b/test/Orleans.Messaging.Tests/Support/BootstrapOutboxServices.cs @@ -80,7 +80,7 @@ public BootstrapOutputGrain(IJournaledStateManager manager, IInbox inbox, Bootst inbox.Register(this); } - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) diff --git a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs index e434a4f0b5c..d30551ed7e5 100644 --- a/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs +++ b/test/Orleans.Messaging.Tests/Support/BootstrapTestGrains.cs @@ -109,7 +109,7 @@ public Task Activate() return Task.CompletedTask; } - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) diff --git a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs index f97fc38b22a..b01d529ec97 100644 --- a/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs +++ b/test/Orleans.Messaging.Tests/Support/RawPayloadTestGrain.cs @@ -73,7 +73,7 @@ public async Task AcceptAndDeactivateAsync(InboxMessage message) return result; } - public bool CanHandle(IInboxHandlerContext context) => true; + public bool CanHandle(InboxMessage message) => true; public async ValueTask HandleAsync(IInboxHandlerContext context, CancellationToken cancellationToken) { From 360c6ba6c867b6e9c69ab8c6605a69a9695207b8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 23:12:12 -0700 Subject: [PATCH 90/95] fix(messaging): copy source-owned messages into outbox dead letters A reference entry's materialized message shares the source's payload and header arrays, and the source may release them in OnRetired. The outbox now copies them into its dead letter before retiring the entry, so the in-memory dead letter and later snapshots keep the original bytes. Inline messages already own their data and are retained without a copy. --- src/Orleans.Messaging/IOutboxMessageSource.cs | 7 +-- src/Orleans.Messaging/Outbox.cs | 17 +++++- src/Orleans.Messaging/README.md | 5 +- .../Functional/MessageSourceAndSinkTests.cs | 52 ++++++++++++++++++- .../RecordingJournalStorageProvider.cs | 15 ++++-- 5 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/Orleans.Messaging/IOutboxMessageSource.cs b/src/Orleans.Messaging/IOutboxMessageSource.cs index 476943ee3da..877d827fac9 100644 --- a/src/Orleans.Messaging/IOutboxMessageSource.cs +++ b/src/Orleans.Messaging/IOutboxMessageSource.cs @@ -64,9 +64,10 @@ public interface IOutboxMessageSource /// How the message retired, and the dead letter's reason for a dead-lettered message. /// /// Stage the release synchronously in journaled state; it commits in the journal write which removes the entry. - /// A dead-lettered message is materialized into the outbox dead letter before this call, unless the source failed to - /// provide it. Don't depend on decoding the payload here: this call also retires a message whose payload the source - /// can no longer provide. An exception fails the activation's messaging, which deactivates the grain. + /// Before this call, the outbox copies a dead-lettered message's payload and header values into its dead letter, + /// unless the source failed to provide the message, so the source can release or reuse its buffers here. Don't depend + /// on decoding the payload here: this call also retires a message whose payload the source can no longer provide. An + /// exception fails the activation's messaging, which deactivates the grain. /// void OnRetired(MessageId messageId, OutboxMessageRetirement retirement); } diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 489dcf8f837..d77c4412816 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -199,6 +199,20 @@ private void Retire(OutboxEntry entry, OutboxMessageRetirement retirement) private static OutboxMessage CreateUnmaterializedMessage(MessageId messageId) => new() { Envelope = new() { MessageId = messageId, Payload = [] }, ReceiverId = default }; + /// Copies the payload and header values of a message which a source owns. + private static OutboxMessage CopySourceMessage(OutboxMessage message) + { + var envelope = message.Envelope; + Dictionary? headers = null; + if (envelope.Headers is { } sourceHeaders) + { + headers = new(sourceHeaders.Count, sourceHeaders.Comparer); + foreach (var (name, value) in sourceHeaders) headers.Add(name, value.AsSpan().ToArray()); + } + + return message with { Envelope = envelope with { Payload = envelope.Payload.AsSpan().ToArray(), Headers = headers } }; + } + private static bool IsUnmaterialized(OutboxMessage message) => message.ReceiverId.IsDefault && message.Envelope is { Payload.Length: 0, Subject: null, Headers: null, DeliverAt: null }; @@ -594,7 +608,8 @@ private void ApplyDeadLetters(Dictionary entries) { var letter = new OutboxDeadLetter { - Message = outcome.Message, + // The source may release a reference entry's payload and headers when it retires, so the dead letter keeps a copy. + Message = outcome.Entry.Reference is null ? outcome.Message : CopySourceMessage(outcome.Message), DeadLetteredAt = now, Reason = error, AttemptCount = attemptCount diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 2d787f43f16..142b74a27bb 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -121,8 +121,9 @@ construction or activation setup, before journal recovery. dead-lettered it) or `DeadLettered` (the outbox dead-lettered it), whose `Reason` carries the dead letter's reason, and whose `Code` carries the code of a `Rejected` message's receiver, while it stages the removal, so the - source's changes commit in the same journal write. An outbox dead letter stores the - materialized message. After the send, a source that throws or returns an invalid + source's changes commit in the same journal write. An outbox dead letter stores its + own copy of the materialized message, so the source can release its payload in + `OnRetired`. After the send, a source that throws or returns an invalid message, such as one with a different ID, dead-letters that entry alone in the same journal write as its `DeadLettered` retirement; the grain keeps processing its other messages. That dead letter carries only the message ID, with an empty payload diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 83401bf85f0..5d8195e8f35 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -127,6 +127,45 @@ public async Task SourceMessage_DeadLetteredByTheOutbox_MaterializesItsMessageBe await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 2); } + [Fact] + public async Task SourceMessage_DeadLetteredByTheOutbox_KeepsItsPayloadAndHeadersWhenTheSourceReleasesThem() + { + var sender = NewGrain(); + var receiver = Cluster.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + var messageId = NewMessageId("released-dead-letter"); + var payload = NewPayload(); + + // The source shares its payload array as a header value and clears that array in OnRetired. + Probe.For(sender).ReleaseBuffersOnRetired = true; + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, payload, write: true); + + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Grain(context).GetState().Retired.Count == 1); + AssertCarriesPayload(Assert.Single(await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters))); + + // The source state journals the payload once, and the dead letter journals it as its payload and its header. + await WaitUntilAsync(sender, _ => Storage.CountPayloadOccurrences(Journal(sender), payload) == 3); + + // A snapshot written after the retirement journals the dead letter's own copy, which a new activation recovers. + Storage.RequestSnapshot(Journal(sender)); + await sender.WriteAsync(); + Assert.Equal(5, Storage.CountPayloadOccurrences(Journal(sender), payload)); + Assert.True(Cluster.TryGetGrainContext(sender.GetGrainId(), out var activation)); + await sender.DeactivateAsync(); + await activation.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), Token); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + AssertCarriesPayload(Assert.Single(await OnGrainAsync(sender, context => + context.ActivationServices.GetRequiredService().OutboxDeadLetters))); + + void AssertCarriesPayload(DeadLetter letter) + { + Assert.Equal(messageId, letter.Message.Envelope.MessageId); + Assert.Equal(payload, letter.Message.Envelope.Payload); + Assert.Equal(payload, Assert.Single(letter.Message.Envelope.Headers!, header => header.Key == SourceSinkGrain.PayloadHeader).Value); + } + } + [Fact] public async Task SourceMessage_DeadLetteredByTheReceiver_RetiresAsRejectedWithTheReceiversReasonAndCode() { @@ -958,6 +997,9 @@ public sealed class Counters private int _failuresRemaining; public volatile SinkMode Mode; + + /// Makes the source share its payload array as a header value, and clear that array when the message retires. + public volatile bool ReleaseBuffersOnRetired; public ConcurrentDictionary Faults { get; } = new(); public int Activations => Volatile.Read(ref _activations); public int Materializations => Volatile.Read(ref _materializations); @@ -994,6 +1036,7 @@ public interface ISourceSinkGrain : IGrainWithGuidKey public sealed class SourceSinkGrain : Grain, ISourceSinkGrain, IMessagingGrain, IOutboxMessageSource, IInboxMessageSink { internal const string Subject = "source-sink/payload"; + internal const string PayloadHeader = "source-sink/payload-copy"; internal const string Rejection = "The sink rejected the message."; internal const string RejectionCode = "source-sink/rejected"; internal const string SinkFailure = "The sink failed before deciding the message's outcome."; @@ -1110,7 +1153,13 @@ bool IOutboxMessageSource.TryGetMessage(MessageId messageId, out OutboxMessage m _probe.OnMaterialized(); message = new() { - Envelope = new() { MessageId = messageId, Payload = payload, Subject = Subject }, + Envelope = new() + { + MessageId = messageId, + Payload = payload, + Subject = Subject, + Headers = _probe.ReleaseBuffersOnRetired ? new() { [PayloadHeader] = payload } : null + }, ReceiverId = GrainId.Parse(_receivers[key]) }; if (_probe.Faults.TryGetValue(messageId, out var fault)) @@ -1129,6 +1178,7 @@ bool IOutboxMessageSource.TryGetMessage(MessageId messageId, out OutboxMessage m void IOutboxMessageSource.OnRetired(MessageId messageId, OutboxMessageRetirement retirement) { var key = messageId.ToString(); + if (_probe.ReleaseBuffersOnRetired) Array.Clear(_outgoing[key]); Assert.True(_outgoing.Remove(key)); Assert.True(_receivers.Remove(key)); _retired[key] = retirement.Status.ToString(); diff --git a/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs index 9797fd59255..e49e1885191 100644 --- a/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs +++ b/test/Orleans.Messaging.Tests/Support/RecordingJournalStorageProvider.cs @@ -10,12 +10,16 @@ namespace Orleans.Messaging.Tests.Support; public sealed class RecordingJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog { private readonly ConcurrentDictionary> _writes = new(); + private readonly ConcurrentDictionary _snapshots = new(); private VolatileJournalStorageProvider? _inner; public void Configure(IOptions options) => _inner ??= new VolatileJournalStorageProvider(options); + /// Requests that the journal's next write replaces it with a snapshot. + public void RequestSnapshot(JournalId journalId) => _snapshots[journalId] = 0; + public IJournalStorage CreateStorage(JournalId journalId) => - new RecordingJournalStorage(_writes.GetOrAdd(journalId, static _ => new()), Inner.CreateStorage(journalId)); + new RecordingJournalStorage(this, journalId, _writes.GetOrAdd(journalId, static _ => new()), Inner.CreateStorage(journalId)); public IAsyncEnumerable ListAsync( JournalCatalogListOptions? options = null, @@ -61,9 +65,13 @@ private static int CountOccurrences(ReadOnlySpan data, ReadOnlySpan private VolatileJournalStorageProvider Inner => _inner ?? throw new InvalidOperationException("The recording journal storage provider has not been configured."); - private sealed class RecordingJournalStorage(ConcurrentQueue writes, IJournalStorage inner) : IJournalStorage + private sealed class RecordingJournalStorage( + RecordingJournalStorageProvider owner, + JournalId journalId, + ConcurrentQueue writes, + IJournalStorage inner) : IJournalStorage { - public bool IsCompactionRequested => inner.IsCompactionRequested; + public bool IsCompactionRequested => owner._snapshots.ContainsKey(journalId) || inner.IsCompactionRequested; public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) => inner.ReadAsync(consumer, cancellationToken); @@ -87,6 +95,7 @@ public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationTo { var bytes = value.ToArray(); await inner.ReplaceAsync(value, cancellationToken).ConfigureAwait(false); + owner._snapshots.TryRemove(journalId, out _); writes.Enqueue(bytes); } From f71ec234ad421acd6a9e986184277d8c21807c25 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 23:12:21 -0700 Subject: [PATCH 91/95] fix(messaging): count unavailable outbox references toward the batch budget Withdrawn and unprovidable source references now count toward OutboxBatchSize, so one pump pass examines and removes at most one batch of entries; later passes remove the rest. --- .../Configuration/DurableMessagingOptions.cs | 4 +++ src/Orleans.Messaging/Outbox.cs | 14 +++++---- .../Functional/MessageSourceAndSinkTests.cs | 29 +++++++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs index cda885888d6..378bcf478f2 100644 --- a/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs +++ b/src/Orleans.Messaging/Configuration/DurableMessagingOptions.cs @@ -161,6 +161,10 @@ public class DurableMessagingOptions /// /// Gets or sets the maximum number of outbox messages processed by one messaging pump pass. /// + /// + /// Each pass delivers, removes, or dead-letters at most this many messages. Messages which their outbox message sources + /// withdrew or failed to provide count toward the limit, and later passes process the remaining messages. + /// public int OutboxBatchSize { get; set; } = 32; /// diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index d77c4412816..11e41db6321 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -308,28 +308,30 @@ private async ValueTask PersistAsync() { var now = TimeProvider.GetUtcNow(); List<(OutboxEntry Entry, OutboxMessage Message)>? candidates = null; + var examined = 0; foreach (var entry in _messages.Values) { if (entry.Sequence <= _durableSequence && IsReadyForAttempt(entry.State, now)) { - OutboxMessage message; try { - if (!TryGetMessage(entry, out message)) + if (TryGetMessage(entry, out var message)) + { + (candidates ??= []).Add((entry, message)); + } + else { (withdrawn ??= []).Add(entry); - continue; } } catch (Exception exception) { // The entry is durable, so a source which can't provide a valid message dead-letters only that entry. (failed ??= []).Add((entry, exception)); - continue; } - (candidates ??= []).Add((entry, message)); - if (candidates.Count == Options.OutboxBatchSize) break; + // Withdrawn and failed entries count toward the batch, so a pass removes at most a batch of entries. + if (++examined == Options.OutboxBatchSize) break; } } if (candidates is null) diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 5d8195e8f35..6e01a250440 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -9,6 +9,7 @@ using Orleans.Hosting; using Orleans.Journaling; using Orleans.Journaling.Json; +using Orleans.Messaging.Configuration; using Orleans.Messaging.Diagnostics; using Orleans.Messaging.Tests.Support; using Orleans.Runtime; @@ -329,6 +330,34 @@ public async Task SourceMessage_WithdrawnBySource_IsRemovedWithoutDeliveryOrReti Assert.Equal(0, await sender.GetOutboxCountAsync()); } + [Fact] + public async Task SourceMessages_WithdrawnBeyondTheBatchSize_AreRemovedOneBatchPerWrite() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + var batchSize = await OnGrainAsync( + sender, + context => context.ActivationServices.GetRequiredService>().Value.OutboxBatchSize); + var messageIds = Enumerable.Range(0, 2 * batchSize + 3).Select(_ => NewMessageId("withdrawn")).ToArray(); + foreach (var messageId in messageIds) + { + await sender.StageFromSourceAsync(receiver.GetGrainId(), messageId, NewPayload(), write: false); + await sender.WithdrawAsync(messageId); + } + + await sender.WriteAsync(); + + // Each removal write removes at most one batch of entries, and later passes remove the rest. + await WaitUntilAsync(sender, context => Outbox(context).Count == 0 && Writes(sender).Length == 4); + var removals = Writes(sender)[1..] + .Select(write => messageIds.Count(messageId => write.AsSpan().IndexOf(Encoding.UTF8.GetBytes(messageId.ToString())) >= 0)) + .ToArray(); + Assert.Equal(new[] { batchSize, batchSize, 3 }, removals); + Assert.Equal(0, Probe.For(receiver).SinkOffers); + Assert.Equal(0, await sender.GetOutboxCountAsync()); + } + [Theory] [InlineData(SourceFault.Throw)] [InlineData(SourceFault.InvalidMessage)] From 32da7fb1705e409c5c0191bfe21f0f78244ae03f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 17 Oct 2026 23:12:39 -0700 Subject: [PATCH 92/95] test(messaging): cover unrelated cancellation, outbox depth, and duplicates on the real outbox An OperationCanceledException not caused by the outbox's own cancellation is retried and dead-lettered as an ordinary delivery failure. The outbox depth metric is released when an activation deactivates or its write fails, and recovery restores it for durable messages only. Equivalent duplicates of a committed message leave its delivery unchanged, and conflicting duplicates are rejected without changing the pending message, before and after commit. --- .../Functional/FaultInjectionTests.cs | 121 ++++++++++++++++++ .../OutboxDeliveryResilienceTests.cs | 23 +++- .../Support/MessagingClusterFixture.cs | 10 ++ .../Support/MessagingMetricProbe.cs | 26 +++- 4 files changed, 171 insertions(+), 9 deletions(-) diff --git a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs index 341b4556a14..3d2104cba5d 100644 --- a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs @@ -431,6 +431,127 @@ public async Task ScheduledDeliveryWakeupSchedulingFailure_IsRetriedBySenderAndP await AssertProcessedOnceAsync(receiver, message); } + [Fact] + public async Task OutboxDepth_IsReleasedByDeactivationAndRestoredByRecovery() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var message = NewMessage(12, "depth-across-activations"); + var id = await sender.SendAsync(receiver.GetGrainId(), Route, message); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + await Fixture.WaitForOutboxDepthAsync(1); + + await _cluster.CrashAsync(sender); + await Fixture.WaitForOutboxDepthAsync(0); + + // Recovery reports the pending message again, and retiring it releases that depth. + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + await Fixture.WaitForOutboxDepthAsync(1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + await Fixture.WaitForOutboxDepthAsync(0); + await AssertProcessedOnceAsync(receiver, message); + } + + [Fact] + public async Task OutboxDepth_AfterAFailedWrite_CountsOnlyDurableMessages() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var durable = NewMessage(13, "durable"); + var id = await sender.SendAsync(receiver.GetGrainId(), Route, durable); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + await Fixture.WaitForOutboxDepthAsync(1); + var previous = Fixture.GetGrainContext(sender); + ArmSenderWriteFailure(sender, ambiguous: false); + + // The failed write fences the activation, which releases the depth of both messages. + await Assert.ThrowsAnyAsync(() => sender.SendAsync(receiver.GetGrainId(), Route, NewMessage(14, "lost"))); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + await Fixture.WaitForOutboxDepthAsync(0); + + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + await Fixture.WaitForOutboxDepthAsync(1); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + await Fixture.WaitForOutboxDepthAsync(0); + await AssertProcessedOnceAsync(receiver, durable); + } + + [Fact] + public async Task EquivalentDuplicate_OfACommittedMessage_LeavesItsDeliveryUnchanged() + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var message = NewMessage(15, "duplicate-after-commit"); + var outgoing = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message); + var id = outgoing.Envelope.MessageId; + await sender.StageOutputAsync(outgoing); + await sender.RetryWriteStateAsync(); + var attempt = await WaitForFailedAttemptAsync(sender, id, 1); + var before = await sender.GetSnapshotAsync(); + + // The grain call delivers a copy of the envelope, which is equivalent to the committed one. + await sender.StageOutputAsync(outgoing); + await sender.RetryWriteStateAsync(); + + Assert.Single(Fixture.GetStagedOutput(sender)); + var unchanged = await Fixture.OnGrainAsync(sender, () => _cluster.GetOutboxState(sender).Attempt(id)); + Assert.Equal(attempt.AttemptCount, unchanged.AttemptCount); + Assert.Equal(attempt.NextAttemptAt, unchanged.NextAttemptAt); + Assert.Equal(attempt.ExpirationTime, unchanged.ExpirationTime); + Assert.Equal(before.ActivationId, (await sender.GetSnapshotAsync()).ActivationId); + + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + Assert.Equal(1, Deliveries.ReceiverCalls); + await AssertProcessedOnceAsync(receiver, message); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ConflictingDuplicate_IsRejectedWithoutChangingThePendingMessage(bool commitFirst) + { + var sender = NewGrain(); + var receiver = NewGrain(); + Deliveries.Enqueue(DeliveryFault.Throw()); + var message = NewMessage(16, "original"); + var outgoing = TestApplicationProtocol.Create(Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message); + var id = outgoing.Envelope.MessageId; + var conflicting = TestApplicationProtocol.Create( + Sessions, sender.GetGrainId(), receiver.GetGrainId(), Route, message with { Value = "conflicting" }); + Assert.Equal(id, conflicting.Envelope.MessageId); + await sender.StageOutputAsync(outgoing); + if (commitFirst) + { + await sender.RetryWriteStateAsync(); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + } + + var before = await sender.GetSnapshotAsync(); + + var error = await Assert.ThrowsAsync(() => sender.StageOutputAsync(conflicting)); + Assert.Contains(id.ToString(), error.Message); + Assert.Equal(outgoing.Envelope.Payload, Assert.Single(Fixture.GetStagedOutput(sender)).Envelope.Payload); + if (!commitFirst) + { + await sender.RetryWriteStateAsync(); + _ = await WaitForFailedAttemptAsync(sender, id, 1); + } + + Assert.Equal(before.ActivationId, (await sender.GetSnapshotAsync()).ActivationId); + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + await WaitForDrainedOutboxAsync(sender); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + var processed = await AssertProcessedOnceAsync(receiver, message); + Assert.Equal(message.Value, Assert.Single(processed.Effects).Value); + } + private void ArmSenderWriteFailure(IMessagingTestGrain sender, bool ambiguous) { var journal = JournalId.FromGrainId(sender.GetGrainId()); diff --git a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs index 37d3abb2568..1d3b6aaf980 100644 --- a/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/OutboxDeliveryResilienceTests.cs @@ -134,6 +134,23 @@ public async Task HandlerNotFound_DeadLettersAfterOutboxRetryTimeout() Assert.Empty((await receiver.GetSnapshotAsync()).Effects); } + [Fact] + public async Task UnrelatedDeliveryCancellation_IsAnOrdinaryFailure() + { + await using var fixture = await StartAsync(Scripted, static options => options.OutboxRetryTimeout = TimeSpan.FromMilliseconds(100)); + fixture.Strategy.Repeat(Step.Throw(new OperationCanceledException("Receiver canceled its operation."))); + + var (sender, receiver) = await SendAsync(fixture, "canceled-by-receiver"); + + var snapshot = await fixture.WaitForDeadLetterCountAsync(sender, 1); + var deadLetter = Assert.Single(snapshot.OutboxDeadLetters); + Assert.Contains(nameof(OperationCanceledException), deadLetter.Reason); + Assert.Contains("Receiver canceled its operation.", deadLetter.Reason); + Assert.True(deadLetter.AttemptCount > 1); + Assert.Equal(0, snapshot.OutboxCount); + Assert.Empty((await receiver.GetSnapshotAsync()).Effects); + } + private static ResiliencePipeline Scripted(ScriptedStrategy strategy) => new ResiliencePipelineBuilder() .AddStrategy(_ => strategy, new ScriptedStrategyOptions()) .Build(); @@ -175,11 +192,13 @@ protected override void ConfigureOptions(DurableMessagingOptions options) } /// A scripted outcome of one try: a receiver answer, or a failure without an answer. - private readonly record struct Step(DeliveryResult? Result, TimeSpan Delay) + private readonly record struct Step(DeliveryResult? Result, TimeSpan Delay, Exception? Error = null) { public static Step Answer(DeliveryResult result, TimeSpan delay = default) => new(result, delay); public static Step Unanswered(TimeSpan delay = default) => new(null, delay); + + public static Step Throw(Exception error) => new(null, default, error); } /// Plays scripted steps in place of calling the receiver, then repeats a step or calls the receiver. @@ -226,7 +245,7 @@ private static async ValueTask> PlayAsync(Step step, C return step.Result is { } result ? Outcome.FromResult((TResult)(object)result) - : Outcome.FromException(new TimeoutException("Injected delivery failure.")); + : Outcome.FromException(step.Error ?? new TimeoutException("Injected delivery failure.")); } } diff --git a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs index 5a026aa617b..b483773b69c 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingClusterFixture.cs @@ -154,6 +154,16 @@ public Task WaitForDeadLetterCountAsync(IMessagingTestGrain gr public Task WaitForOutboxCountAsync(IMessagingTestGrain grain, int expected) => SnapshotProbe.WaitAsync(grain.GetGrainId(), snapshot => snapshot.OutboxCount == expected); + /// Gets the outbox depth which the cluster's silos report through the outbox depth metric. + public long GetOutboxDepth() => + Cluster.Silos.Sum(silo => Metrics.GetDepth( + silo.ServiceProvider.GetRequiredService().Meter, + "orleans-messaging-outbox-depth")); + + /// Waits until the cluster's silos report an outbox depth of . + public Task WaitForOutboxDepthAsync(long expected) => + Metrics.WaitUntilAsync(() => GetOutboxDepth() == expected, TestContext.Current.CancellationToken); + public EndpointSnapshot GetSnapshot(IMessagingTestGrain grain) => GetGrainInstance(grain).GetSnapshotForTest(); diff --git a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs index defbd51d690..7b06473c373 100644 --- a/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs +++ b/test/Orleans.Messaging.Tests/Support/MessagingMetricProbe.cs @@ -7,7 +7,7 @@ namespace Orleans.Messaging.Tests.Support; public sealed class MessagingMetricProbe : IDisposable { private readonly ConcurrentDictionary<(string Instrument, string TagValue), long> _measurements = []; - private readonly ConcurrentDictionary _depths = []; + private readonly ConcurrentDictionary<(Meter Meter, string Instrument), long> _depths = []; private readonly object _lock = new(); private TaskCompletionSource _changed = CreateSignal(); private readonly MeterListener _listener; @@ -39,7 +39,9 @@ public MessagingMetricProbe() public long GetCount(string instrument, string tagValue = "") => _measurements.TryGetValue((instrument, tagValue), out var count) ? count : 0; - public long GetDepth(string instrument) => _depths.TryGetValue(instrument, out var value) ? value : 0; + /// Gets the current value of a depth instrument created by . + /// Each silo creates its own meter, so a meter identifies one silo's depth. + public long GetDepth(Meter meter, string instrument) => _depths.TryGetValue((meter, instrument), out var value) ? value : 0; public Task WaitForCountAsync( string instrument, @@ -51,20 +53,24 @@ public Task WaitForCountAsync( tagValue, TestContext.Current.CancellationToken); - public async Task WaitForCountWithCancellationAsync( + public Task WaitForCountWithCancellationAsync( string instrument, long expected, string tagValue, - CancellationToken cancellationToken) + CancellationToken cancellationToken) => + WaitUntilAsync(() => GetCount(instrument, tagValue) >= expected, cancellationToken); + + /// Waits until , which reads this probe's measurements, holds. + public async Task WaitUntilAsync(Func condition, CancellationToken cancellationToken) { using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeout.CancelAfter(TimeSpan.FromSeconds(30)); - while (GetCount(instrument, tagValue) < expected) + while (!condition()) { Task changed; lock (_lock) { - if (GetCount(instrument, tagValue) >= expected) + if (condition()) { return; } @@ -87,7 +93,8 @@ private void OnMeasurement( if (instrument.Name is "orleans-messaging-inbox-depth" or "orleans-messaging-outbox-depth") { // The depth instruments are UpDownCounters, so each measurement is a delta. - _depths.AddOrUpdate(instrument.Name, measurement, (_, current) => current + measurement); + _depths.AddOrUpdate((instrument.Meter, instrument.Name), measurement, (_, current) => current + measurement); + SignalChanged(); return; } @@ -106,6 +113,11 @@ private void OnMeasurement( (instrument.Name, tagValue), measurement, (_, current) => current + measurement); + SignalChanged(); + } + + private void SignalChanged() + { lock (_lock) { _changed.TrySetResult(); From 26a2c376f12944c883304697c0fd3a3cc428421c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 18 Oct 2026 00:17:52 -0700 Subject: [PATCH 93/95] fix(messaging): answer duplicates of sink-rejected messages with their rejection An inbox sink's rejection completes the message, so a redelivery within DeduplicationWindow was answered Duplicate, which the sender's outbox retires as Delivered. A sender whose first answer was lost, such as one that deactivated before journaling the retirement, therefore retired a rejected message as delivered. The processed record now keeps the sink's reason and code, through journal entries and snapshots, so the inbox answers duplicates with the original DeadLettered result while the record is retained, independently of the dead letter's retention. --- .../Functional/FaultInjectionTests.cs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs index 3d2104cba5d..07e4acdf785 100644 --- a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs @@ -116,6 +116,56 @@ public async Task LostResponse_AfterTheSinkAccepted_RetiresTheSourceMessageAsDel Assert.Equal(1, probe.SinkAcceptances); } + [Theory] + [InlineData(ReceiverRecovery.None)] + [InlineData(ReceiverRecovery.Journal)] + [InlineData(ReceiverRecovery.Snapshot)] + public async Task LostResponse_AfterTheSinkRejected_RetiresTheSourceMessageAsRejected(ReceiverRecovery recovery) + { + var sender = NewSinkGrain(); + var receiver = NewSinkGrain(); + var probe = _cluster.SourceSinkProbe.For(receiver); + probe.Mode = SinkMode.DeadLetter; + + // The rejection's write is the receiver's first, so a requested snapshot holds its processed record. + if (recovery == ReceiverRecovery.Snapshot) Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(receiver.GetGrainId())); + var lost = DeliveryFault.LoseResponse(); + Deliveries.Enqueue(lost); + var id = NewSourceMessageId("lost-rejection"); + + await sender.StageFromSourceAsync(receiver.GetGrainId(), id, RandomNumberGenerator.GetBytes(32), write: true); + + var rejection = await lost.ReceiverAnswer; + Assert.Equal(DeliveryStatus.DeadLettered, rejection.Status); + await WaitForFailedAttemptAsync(sender, id, 1); + if (recovery != ReceiverRecovery.None) + { + var previous = Fixture.GetGrainContext(receiver); + await receiver.DeactivateAsync(); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + } + + // The receiver answers the retry from its processed record, without offering the message to the sink again. + probe.Mode = SinkMode.Accept; + await _cluster.AdvanceAndRunPumpAsync(sender, RetryInterval); + + await Fixture.WaitUntilAsync(sender, () => Sink(sender).GetState().Retired.Count == 1); + var (retired, reasons, codes) = await OnSinkAsync(sender, static grain => (grain.GetState().Retired, grain.GetRetirementReasons(), grain.GetRetirementCodes())); + Assert.Equal(nameof(OutboxMessageRetirementStatus.Rejected), retired[id.ToString()]); + Assert.Equal(SourceSinkGrain.Rejection, reasons[id.ToString()]); + Assert.Equal(SourceSinkGrain.RejectionCode, codes[id.ToString()]); + Assert.Equal([DeliveryStatus.DeadLettered, DeliveryStatus.DeadLettered], Deliveries.Answers); + Assert.Equal(1, probe.SinkOffers); + Assert.Equal(0, probe.SinkAcceptances); + } + + public enum ReceiverRecovery + { + None, + Journal, + Snapshot + } + [Fact] public async Task TimedOutDelivery_WhoseCallIsStillCommitting_IsDeduplicatedAgainstTheRetry() { From 1032c720893d29f43480f6dcda90eb65ad8db85b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 18 Oct 2026 17:44:19 -0700 Subject: [PATCH 94/95] fix(messaging): retain outbox dead letters on the grain clock --- src/Orleans.Messaging/Outbox.cs | 12 +- src/Orleans.Messaging/README.md | 4 +- .../Functional/MessageSourceAndSinkTests.cs | 2 + .../Functional/ScheduledDeliveryTests.cs | 138 ++++++++++++++++++ 4 files changed, 149 insertions(+), 7 deletions(-) diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 11e41db6321..1ca6ca85824 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -37,8 +37,8 @@ public Outbox(MessagingPump pump) private MessagingShared Shared => _pump.Shared; private DurableMessagingOptions Options => _pump.Shared.Options; private ILogger Logger => _pump.Shared.OutboxLogger; - // Delivery retries, expiry, and retention follow the durable jobs clock which schedules outbox wake-ups. - private TimeProvider TimeProvider => _pump.Shared.JobTimeProvider; + private TimeProvider TimeProvider => Shared.TimeProvider; + private TimeProvider JobTimeProvider => Shared.JobTimeProvider; private GrainId GrainId => _pump.GrainContext.GrainId; internal IReadOnlyDictionary MessageState => _messages; @@ -306,7 +306,7 @@ private async ValueTask PersistAsync() ref List? withdrawn, ref List<(OutboxEntry Entry, Exception Error)>? failed) { - var now = TimeProvider.GetUtcNow(); + var now = JobTimeProvider.GetUtcNow(); List<(OutboxEntry Entry, OutboxMessage Message)>? candidates = null; var examined = 0; foreach (var entry in _messages.Values) @@ -345,7 +345,7 @@ private async ValueTask PersistAsync() private async Task DeliverAsync(OutboxEntry entry, OutboxMessage message, CancellationToken cancellationToken) { - var startedAt = TimeProvider.GetUtcNow(); + var startedAt = JobTimeProvider.GetUtcNow(); if (IsRetryExpired(entry.State, startedAt)) { return new(entry, message, null, entry.State.LastError, TimeSpan.Zero, startedAt, true); @@ -589,7 +589,7 @@ private void ApplyDeadLetters(Dictionary entries) { var previous = outcome.Entry.State; var status = outcome.Result?.Status; - var now = TimeProvider.GetUtcNow(); + var now = JobTimeProvider.GetUtcNow(); var attemptCount = outcome.Expired ? previous.AttemptCount : checked(previous.AttemptCount + 1); // A receiver checks its deduplication records before it answers, so a Backpressured answer proves that no earlier @@ -612,7 +612,7 @@ private void ApplyDeadLetters(Dictionary entries) { // The source may release a reference entry's payload and headers when it retires, so the dead letter keeps a copy. Message = outcome.Entry.Reference is null ? outcome.Message : CopySourceMessage(outcome.Message), - DeadLetteredAt = now, + DeadLetteredAt = TimeProvider.GetUtcNow(), Reason = error, AttemptCount = attemptCount }; diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 142b74a27bb..4498e90f811 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -188,7 +188,8 @@ and later batches deliver other messages meanwhile. The outbox retries `Backpressured` until the receiver takes the message. It logs `HandlerNotFound` as an error. Other failures, including calls which fail without an answer during an outage, dead-letter the message after `OutboxRetryTimeout` (6 hours) from the first -failed attempt since the last `Backpressured` answer. Each attempt runs through the +failed attempt since the last `Backpressured` answer. Retry deadlines and backoff +follow the Durable Jobs clock. Each attempt runs through the `DeliveryResiliencePipeline` Polly pipeline, which can add timeouts, retries, or a circuit breaker. Delivery order is unspecified. @@ -206,6 +207,7 @@ state. After the grain's state is deleted, both start empty and continue working Completed message IDs are retained for `DeduplicationWindow`. Dead letters are retained up to `DeadLetterRetentionPeriod` and `MaxRetainedDeadLetters` for each queue. +Dead-letter timestamps and retention follow the grain's `TimeProvider`. ## Diagnostics diff --git a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs index 6e01a250440..a7a2f02369b 100644 --- a/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/MessageSourceAndSinkTests.cs @@ -390,6 +390,7 @@ public async Task SourceFailureAfterSend_DeadLettersOnlyThatEntryAndKeepsTheGrai var received = (await OnGrainAsync(receiver, context => Grain(context).GetState())).Received; Assert.Equal(new[] { first.ToString(), second.ToString() }.Order(), received.Keys.Order()); AssertSourceFailureDeadLetter(Assert.Single(deadLetters), failing, fault); + Assert.Equal(fixture.Clock.GetUtcNow(), Assert.Single(deadLetters).DeadLetteredAt); Assert.Equal( Assert.Single(deadLetters).Reason, Assert.Single(await OnGrainAsync(sender, context => Grain(context).GetRetirementReasons())).Value); @@ -427,6 +428,7 @@ public async Task SourceFailureAfterSend_DeadLettersOnlyThatEntryAndKeepsTheGrai var recovered = await OnGrainAsync(sender, context => context.ActivationServices.GetRequiredService().OutboxDeadLetters); AssertSourceFailureDeadLetter(Assert.Single(recovered), failing, fault); + Assert.Equal(fixture.Clock.GetUtcNow(), Assert.Single(recovered).DeadLetteredAt); Assert.Equal(2, Probe.For(sender).Activations); Assert.Equal(8, Probe.For(sender).Materializations); } diff --git a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs index 2ae0d06a749..1a00e7b537f 100644 --- a/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/ScheduledDeliveryTests.cs @@ -1,8 +1,12 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using Orleans.DurableJobs; using Orleans.Journaling; +using Orleans.Messaging.Configuration; +using Orleans.Messaging.Diagnostics; using Orleans.Messaging.Tests.Support; using Orleans.Runtime; +using Orleans.TestingHost.Diagnostics; using Xunit; namespace Orleans.Messaging.Tests.Functional; @@ -18,6 +22,140 @@ public sealed class ScheduledDeliveryTests() : MessagingBehaviorTestBase(new Sch private ManualJobClock JobClock => ((ScheduledDeliveryClusterFixture)Fixture).JobClock; + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OutboxRetries_UseJobTimeAndDeadLettersUseGrainTime(bool snapshot) + { + Fixture.Clock.Advance(TimeSpan.FromDays(2)); + var sender = NewGrain(); + var receiver = Fixture.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + _ = await sender.GetSnapshotAsync(); + var options = Fixture.GetGrainContext(sender).ActivationServices + .GetRequiredService>().Value; + using var events = new DiagnosticEventCollector(MessagingEvents.ListenerName); + var firstRetry = WaitForRetryAsync(1); + var startedAt = JobClock.GetUtcNow(); + var expiresAt = startedAt + options.OutboxRetryTimeout; + + var messageId = await sender.SendAsync(receiver.GetGrainId(), "messages/unhandled", NewMessage(61, "dead-letter")); + await firstRetry; + await AssertWakeupAtAsync(sender, startedAt + options.RetryInitialInterval); + var state = await GetRetryStateAsync(); + Assert.Equal(1, state.AttemptCount); + Assert.Equal(expiresAt, state.ExpirationTime); + Assert.Equal(startedAt + options.RetryInitialInterval, state.NextAttemptAt); + + Fixture.Clock.Advance(TimeSpan.FromDays(1)); + await AssertWakeupAtAsync(sender, Assert.IsType(state.NextAttemptAt)); + Assert.Equal(1, (await GetRetryStateAsync()).AttemptCount); + + var secondRetry = WaitForRetryAsync(2); + JobClock.Advance(options.RetryInitialInterval); + _ = await Fixture.RunPumpAsync(sender); + await secondRetry; + state = await GetRetryStateAsync(); + Assert.Equal(expiresAt, state.ExpirationTime); + Assert.Equal(JobClock.GetUtcNow() + 2 * options.RetryInitialInterval, state.NextAttemptAt); + await AssertWakeupAtAsync(sender, Assert.IsType(state.NextAttemptAt)); + + if (snapshot) + { + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(sender.GetGrainId())); + await sender.RetryWriteStateAsync(); + } + await ReactivateAsync(sender); + var recovered = await GetRetryStateAsync(); + Assert.Equal(state.AttemptCount, recovered.AttemptCount); + Assert.Equal(state.ExpirationTime, recovered.ExpirationTime); + Assert.Equal(state.NextAttemptAt, recovered.NextAttemptAt); + await AssertWakeupAtAsync(sender, Assert.IsType(recovered.NextAttemptAt)); + + var thirdRetry = WaitForRetryAsync(3); + JobClock.Advance(expiresAt - JobClock.GetUtcNow() - TimeSpan.FromTicks(1)); + _ = await Fixture.RunPumpAsync(sender); + await thirdRetry; + await AssertWakeupAtAsync(sender, expiresAt); + JobClock.Advance(TimeSpan.FromTicks(1)); + _ = await Fixture.RunPumpAsync(sender); + var letter = Assert.Single((await Fixture.WaitForDeadLetterCountAsync(sender, 1)).OutboxDeadLetters); + Assert.Equal(messageId, letter.MessageId); + Assert.Equal(3, letter.AttemptCount); + Assert.Equal(Fixture.Clock.GetUtcNow(), letter.DeadLetteredAt); + + if (snapshot) + { + Fixture.Storage.RequestSnapshot(JournalId.FromGrainId(sender.GetGrainId())); + await sender.RetryWriteStateAsync(); + } + JobClock.Advance(options.DeadLetterRetentionPeriod); + await ReactivateAsync(sender); + Assert.Equal(letter, Assert.Single((await sender.GetSnapshotAsync()).OutboxDeadLetters)); + + Fixture.Clock.Advance(options.DeadLetterRetentionPeriod - TimeSpan.FromTicks(1)); + await ReactivateAsync(sender); + Assert.Equal(letter, Assert.Single((await sender.GetSnapshotAsync()).OutboxDeadLetters)); + Fixture.Clock.Advance(TimeSpan.FromTicks(1)); + await ReactivateAsync(sender); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + await ReactivateAsync(sender); + Assert.Empty((await sender.GetSnapshotAsync()).OutboxDeadLetters); + + Task WaitForRetryAsync(int attempt) => events.WaitForEventAsync( + nameof(MessagingEvents.OutboxRetryScheduled), + entry => entry.Payload is MessagingEvents.OutboxRetryScheduled scheduled + && scheduled.GrainContext.GrainId == sender.GetGrainId() && scheduled.AttemptCount == attempt, + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + + Task GetRetryStateAsync() => Fixture.OnGrainAsync(sender, () => + Fixture.GetGrainContext(sender).ActivationServices.GetRequiredService().MessageState[messageId].State); + } + + [Fact] + public async Task OutboxDeadLetterInsertion_CompactsOnGrainTime() + { + Fixture.Clock.Advance(TimeSpan.FromDays(2)); + var sender = NewGrain(); + var receiver = Fixture.Client.GetGrain(Guid.NewGuid()); + await receiver.PingAsync(); + _ = await sender.GetSnapshotAsync(); + var options = Fixture.GetGrainContext(sender).ActivationServices + .GetRequiredService>().Value; + + var first = Assert.Single((await DeadLetterAsync(62)).OutboxDeadLetters); + Assert.Equal(Fixture.Clock.GetUtcNow(), first.DeadLetteredAt); + JobClock.Advance(options.DeadLetterRetentionPeriod); + var retained = (await DeadLetterAsync(63)).OutboxDeadLetters; + Assert.Equal(2, retained.Count); + Assert.Contains(first, retained); + Assert.All(retained, letter => Assert.Equal(Fixture.Clock.GetUtcNow(), letter.DeadLetteredAt)); + + Fixture.Clock.Advance(options.DeadLetterRetentionPeriod); + var replacement = Assert.Single((await DeadLetterAsync(64)).OutboxDeadLetters); + Assert.Equal(Fixture.Clock.GetUtcNow(), replacement.DeadLetteredAt); + await ReactivateAsync(sender); + Assert.Equal(replacement, Assert.Single((await sender.GetSnapshotAsync()).OutboxDeadLetters)); + + async Task DeadLetterAsync(int sequence) + { + using var events = new DiagnosticEventCollector(MessagingEvents.ListenerName); + var retry = events.WaitForEventAsync( + nameof(MessagingEvents.OutboxRetryScheduled), + entry => entry.Payload is MessagingEvents.OutboxRetryScheduled scheduled + && scheduled.GrainContext.GrainId == sender.GetGrainId(), + TimeSpan.FromSeconds(30), + TestContext.Current.CancellationToken); + var messageId = await sender.SendAsync(receiver.GetGrainId(), "messages/unhandled", NewMessage(sequence, "dead-letter")); + await retry; + JobClock.Advance(options.OutboxRetryTimeout); + _ = await Fixture.RunPumpAsync(sender); + return await Fixture.SnapshotProbe.WaitAsync(sender.GetGrainId(), + state => state.OutboxCount == 0 && state.OutboxDeadLetters.Any(letter => letter.MessageId == messageId)); + } + } + [Fact] public async Task ScheduledMessage_IsHeldUntilDeliverAtThenProcessedOnce() { From 9cd1916ae45edec1547f76b32c5930aeeee2ace6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 18 Oct 2026 18:03:10 -0700 Subject: [PATCH 95/95] fix(messaging): preserve durable wakeups across journal boundaries --- src/Orleans.Messaging/MessagingPump.cs | 2 + src/Orleans.Messaging/Outbox.cs | 13 ++- src/Orleans.Messaging/README.md | 2 + .../Functional/FaultInjectionTests.cs | 101 ++++++++++++++++++ .../Functional/InboxJournalRecoveryTests.cs | 34 ++++++ 5 files changed, 147 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Messaging/MessagingPump.cs b/src/Orleans.Messaging/MessagingPump.cs index 4a87311c78b..c2ef94afce4 100644 --- a/src/Orleans.Messaging/MessagingPump.cs +++ b/src/Orleans.Messaging/MessagingPump.cs @@ -158,6 +158,8 @@ public async ValueTask ExecuteJobAsync(IJobRunContext conte try { await pass.Task.WaitAsync(cancellationToken).ConfigureAwait(true); + // A failed pass can leave only staged removals; its job must survive the activation's failure. + _shutdown.Token.ThrowIfCancellationRequested(); } catch { diff --git a/src/Orleans.Messaging/Outbox.cs b/src/Orleans.Messaging/Outbox.cs index 1ca6ca85824..c9dbafcf28a 100644 --- a/src/Orleans.Messaging/Outbox.cs +++ b/src/Orleans.Messaging/Outbox.cs @@ -687,19 +687,22 @@ private DateTimeOffset GetNextAttemptAt(OutboxMessageState state, DateTimeOffset } /// - /// Returns the delay until the next delivery attempt is due, or when the outbox is empty. + /// Returns the delay until the next captured delivery attempt is due, or when none remain. /// /// The current Durable Jobs time. internal TimeSpan? GetNextWakeupDelay(DateTimeOffset now) { - if (Count == 0) return null; - var next = DateTimeOffset.MaxValue; + TimeSpan? next = null; foreach (var entry in _messages.Values) { + // Capture follows synchronous preparation; its wake-up covers the write until acknowledgement. + if (entry.Sequence > _capturedSequence) continue; var at = GetNextAttemptAt(entry.State, now); - if (at < next) next = at; + var delay = at > now ? at - now : TimeSpan.Zero; + if (delay == TimeSpan.Zero) return delay; + if (next is null || delay < next) next = delay; } - return next > now ? next - now : TimeSpan.Zero; + return next; } internal async Task StartAsync(CancellationToken cancellationToken) diff --git a/src/Orleans.Messaging/README.md b/src/Orleans.Messaging/README.md index 4498e90f811..5c134b8bc64 100644 --- a/src/Orleans.Messaging/README.md +++ b/src/Orleans.Messaging/README.md @@ -197,6 +197,8 @@ Each activation runs one messaging pump on its scheduler, processing inbox batch and delivering outbox batches. One durable job per grain wakes the pump while either queue or a registered work source holds work. Journaled state determines which messages are due, so each wake-up processes exactly the ready work. +In-flight delivery calls retain a durable wake-up so recovered activations can +resume journaled messages. ## State diff --git a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs index 07e4acdf785..7992010cbb4 100644 --- a/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/FaultInjectionTests.cs @@ -38,6 +38,102 @@ public FaultInjectionTests() : this(new FaultInjectionClusterFixture()) private DeliveryFaultInjector Deliveries => _cluster.Deliveries; + [Fact] + public async Task OutboxWakeup_BareStagingWaitsForCommitAfterPriorDeliveryDrains() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + await sender.SendAsync(receiver.GetGrainId(), Route, NewMessage(20, "committed")); + await Fixture.OnGrainAsync(sender, () => Fixture.GetPump(sender).EnsureStarted()); + await WaitForDrainedOutboxAsync(sender); + var pump = Fixture.GetPump(sender); + Assert.NotNull(await Fixture.OnGrainAsync(sender, () => InboxTestAccess.WakeupJobId(pump))); + var attempts = Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId()); + await sender.StageWithoutCommitAsync(receiver.GetGrainId(), Route, NewMessage(21, "staged")); + + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(sender)).Status); + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(sender)).Status); + Assert.Equal(attempts, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId())); + Assert.Equal(1, (await sender.GetSnapshotAsync()).OutboxCount); + Assert.Equal(1, Deliveries.ReceiverCalls); + + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + await sender.RetryWriteStateAsync(); + Assert.Equal(attempts + 1, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId())); + await WaitForDrainedOutboxAsync(sender); + var completed = await Fixture.WaitForEffectCountAsync(receiver, 2); + Assert.Equal(new[] { "committed", "staged" }, completed.Effects.Select(effect => effect.Value)); + Assert.All(completed.Effects, effect => Assert.Equal(1, effect.Count)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OutboxWakeup_CapturedWriteRetainsCoverageUntilAcknowledgedAndRecovered(bool snapshot) + { + var sender = NewGrain(); + var receiver = NewGrain(); + var before = await sender.GetSnapshotAsync(); + var previous = Fixture.GetGrainContext(sender); + var journal = JournalId.FromGrainId(sender.GetGrainId()); + if (snapshot) Fixture.Storage.RequestSnapshot(journal); + using var acknowledgement = Fixture.Storage.BlockAcknowledgement(journal); + var message = NewMessage(22, "captured"); + var sending = sender.SendAsync(receiver.GetGrainId(), Route, message); + await acknowledgement.WaitUntilEnteredAsync(); + Assert.False(sending.IsCompleted); + Assert.Equal(0, Deliveries.ReceiverCalls); + + // Captured entries retain a wake-up while delivery awaits the write acknowledgement. + var wakeup = await Fixture.RunPumpAsync(sender); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, wakeup.Status); + Assert.Equal(_cluster.JobClock.GetUtcNow() + TimeSpan.FromSeconds(1), wakeup.RescheduleTime); + Assert.Equal(0, Deliveries.ReceiverCalls); + + acknowledgement.Fail(); + await Assert.ThrowsAsync(() => sending); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + var recovered = await sender.GetSnapshotAsync(); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Equal(1, Deliveries.ReceiverCalls); + Assert.Equal([DeliveryStatus.Accepted], Deliveries.Answers); + } + + [Fact] + public async Task OutboxWakeup_CompletedJobDuringPreparationIsReplacedBeforeCapture() + { + var sender = NewGrain(); + var receiver = NewGrain(); + _ = await sender.GetSnapshotAsync(); + var journal = JournalId.FromGrainId(sender.GetGrainId()); + using var write = Fixture.Storage.BlockWrite(journal); + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + using var scheduling = Fixture.JobManagerProbe.BlockNext(MessagingPump.JobName); + var message = NewMessage(23, "prepared"); + var sending = sender.SendAsync(receiver.GetGrainId(), Route, message); + await scheduling.WaitUntilEnteredAsync(); + + Assert.Equal(DurableJobRunStatus.Completed, (await Fixture.RunPumpAsync(sender)).Status); + Assert.False(sending.IsCompleted); + Assert.Equal(0, Fixture.Storage.GetSuccessfulWriteCount(journal)); + scheduling.Continue(); + await write.WaitUntilEnteredAsync(); + Assert.Equal(2, Fixture.JobManagerProbe.GetAttemptCount(MessagingPump.JobName, sender.GetGrainId())); + Assert.Single(Fixture.JobManagerProbe.GetScheduledJobs(MessagingPump.JobName, sender.GetGrainId())); + Assert.Equal(0, Deliveries.ReceiverCalls); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, (await Fixture.RunPumpAsync(sender)).Status); + + write.Release(); + await sending; + await WaitForDrainedOutboxAsync(sender); + await AssertProcessedOnceAsync(receiver, message); + Assert.Equal(1, Deliveries.ReceiverCalls); + } + [Theory] [InlineData(DeliveryFaultKind.Throw, "Injected transient delivery failure.")] [InlineData(DeliveryFaultKind.TimeOut, "TimeoutRejectedException")] @@ -370,6 +466,11 @@ public async Task SenderCrash_AfterTheReceiverCommitted_RedeliversAsADuplicate() await hold.WaitUntilEnteredAsync(); Assert.Equal(DeliveryStatus.Accepted, (await hold.ReceiverAnswer).Status); + var wakeup = await Fixture.RunPumpAsync(sender); + Assert.Equal(DurableJobRunStatus.RescheduleRequested, wakeup.Status); + Assert.Equal(_cluster.JobClock.GetUtcNow() + TimeSpan.FromSeconds(1), wakeup.RescheduleTime); + Assert.Equal(1, Deliveries.ReceiverCalls); + // Deactivation cancels the in-flight delivery, whose outcome is never applied or counted as an attempt. await _cluster.CrashAsync(sender); _ = await sender.GetSnapshotAsync(); diff --git a/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs b/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs index f85dce575de..c58a0e785fa 100644 --- a/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs +++ b/test/Orleans.Messaging.Tests/Functional/InboxJournalRecoveryTests.cs @@ -93,4 +93,38 @@ public async Task WakeupSchedulingFailure_RetriedDeliveryProcessesOnce() var completed = await Fixture.WaitForEffectCountAsync(receiver, 1); Assert.Equal(1, Assert.Single(completed.Effects).Count); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompletionWriteFailure_CancelsTheOwningJobPassBeforeRecovery(bool ambiguous) + { + var receiver = NewGrain(); + var before = await receiver.GetSnapshotAsync(); + var previous = Fixture.GetGrainContext(receiver); + const string route = "messages/failed-job-pass"; + using var handler = Fixture.HandlerProbe.Arm(receiver.GetGrainId(), route); + var envelope = CreateEnvelope(receiver, NewMessage(77, "failed-job-pass"), route); + Fixture.JobManagerProbe.SkipNext(MessagingPump.JobName); + await Fixture.OnGrainAsync(receiver, () => Fixture.GetInbox(receiver).StageLocal(envelope)); + await receiver.RetryWriteStateAsync(); + + // This job owns the pass which enters the handler. + var running = Fixture.RunPumpAsync(receiver); + await handler.WaitUntilEnteredAsync(); + var journal = JournalId.FromGrainId(receiver.GetGrainId()); + if (ambiguous) Fixture.Storage.FailAfterWrite(journal); + else Fixture.Storage.FailWrite(journal); + handler.Release(); + + await Assert.ThrowsAnyAsync(() => running); + await previous.Deactivated.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); + _ = await receiver.GetSnapshotAsync(); + var recovered = await Fixture.WaitForEffectCountAsync(receiver, 1); + Assert.NotEqual(before.ActivationId, recovered.ActivationId); + Assert.Equal("failed-job-pass", Assert.Single(recovered.Effects).Value); + Assert.Equal(1, Assert.Single(recovered.Effects).Count); + Assert.Equal(0, recovered.InboxCount); + Assert.Empty(recovered.InboxDeadLetters); + } }