From 45cf296f27810143f3c1fb9b1583e528c74ea385 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 28 Aug 2026 16:33:35 +0200 Subject: [PATCH 001/136] Retain completion facts instead of the command in the message enumerator --- Slon/Pg/Protocol/Flows/CommandExtensions.cs | 16 ++++++++--- .../Flows/CommandFlow.MessageEnumerator.cs | 28 ++++++++++--------- Slon/Pg/Protocol/Flows/CommandFlow.cs | 2 +- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandExtensions.cs b/Slon/Pg/Protocol/Flows/CommandExtensions.cs index 564294e..4c09b5b 100644 --- a/Slon/Pg/Protocol/Flows/CommandExtensions.cs +++ b/Slon/Pg/Protocol/Flows/CommandExtensions.cs @@ -533,9 +533,14 @@ public static (PgError?, ParameterTypeList, RowDescription?) ReadPreparationDesc /// If more commands before the next Sync are expected these would be discarded and absent from the message stream. /// In case of an error inside an explicit transaction block all commands until rollback are affected. public static (PgError, TransactionStatus)? Complete(this in Command command, PgDecoder decoder) + => Complete(command.DescribeOnly, command.WithSync, decoder); + + // Completion depends only on whether an Execute was written and whether a Sync follows it, so a + // reader can retain those two facts instead of the command. + internal static (PgError, TransactionStatus)? Complete(bool describeOnly, bool withSync, PgDecoder decoder) { PgError? errorMessage = null; - if (!command.DescribeOnly) + if (!describeOnly) { // https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY // "Therefore, an Execute phase is always terminated by the appearance of exactly one of these messages: @@ -544,7 +549,7 @@ public static (PgError, TransactionStatus)? Complete(this in Command command, Pg PgTypes.BackendType.CommandComplete, PgTypes.BackendType.EmptyQueryResponse, PgTypes.BackendType.PortalSuspended); } - if (!command.WithSync) + if (!withSync) return errorMessage is not null ? (errorMessage, TransactionStatus.Unknown) : null; // Reading the following RFQ may retire the batch which owns the ErrorResponse body. @@ -565,9 +570,12 @@ public static (PgError, TransactionStatus)? Complete(this in Command command, Pg /// If more commands before the next Sync are expected these would be discarded and absent from the message stream. /// In case of an error inside an explicit transaction block all commands until rollback are affected. public static ValueTask<(PgError, TransactionStatus)?> CompleteAsync(this in Command command, PgDecoder decoder) + => CompleteAsync(command.DescribeOnly, command.WithSync, decoder); + + internal static ValueTask<(PgError, TransactionStatus)?> CompleteAsync(bool describeOnly, bool withSync, PgDecoder decoder) { PgError? errorMessage = null; - if (!command.DescribeOnly) + if (!describeOnly) { // https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY // "Therefore, an Execute phase is always terminated by the appearance of exactly one of these messages: @@ -576,7 +584,7 @@ public static (PgError, TransactionStatus)? Complete(this in Command command, Pg PgTypes.BackendType.CommandComplete, PgTypes.BackendType.EmptyQueryResponse, PgTypes.BackendType.PortalSuspended); } - if (!command.WithSync) + if (!withSync) return errorMessage is not null ? new((errorMessage, TransactionStatus.Unknown)) : new(result: null); // Reading the following RFQ may retire the batch which owns the ErrorResponse body. diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 971fa22..3dc0eca 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -56,8 +56,8 @@ internal readonly struct ResultMessageEnumerator() : IEnumerator BackendMessage IEnumerator.Current => _messageEnumerator.Current; object? IEnumerator.Current => ((IEnumerator)_messageEnumerator).Current; - public void Initialize(CommandFlow flow, PgDecoder decoder) - => _messageEnumerator.Initialize(flow, decoder); + public void Initialize(in Command command, PgDecoder decoder) + => _messageEnumerator.Initialize(command, decoder); public void Reset() => _messageEnumerator.Reset(); @@ -66,7 +66,10 @@ public void Initialize(CommandFlow flow, PgDecoder decoder) sealed class MessageEnumerator : IEnumerator, IAsyncEnumerator { - CommandFlow _flow = null!; + // Completion needs only these two command facts. Retaining them keeps the protocol-static + // enumerator independent of the flow and avoids holding a reference-bearing command copy. + bool _describeOnly; + bool _withSync; PgDecoder _decoder = null!; bool _disposed; bool _first; @@ -74,8 +77,6 @@ sealed class MessageEnumerator : IEnumerator, IAsyncEnumerator _flow._commands[_flow._commandIndex]; - // An Execute response consists of DataRow messages followed by one terminal message. [Conditional("DEBUG")] static void DebugEnsureExpected(BackendMessage message) @@ -197,7 +198,7 @@ public void Dispose() { while (decoder.GetNext().Header.Type is PgTypes.BackendType.DataRow) {} } - _completeError = Command.Complete(_decoder); + _completeError = CommandExtensions.Complete(_describeOnly, _withSync, _decoder); } catch (Exception ex) { @@ -224,7 +225,7 @@ ValueTask DisposeAsyncCore() if (decoder.TryGetCurrent(out var current) && current.Header.Type is not PgTypes.BackendType.DataRow) { - var completion = Command.CompleteAsync(decoder); + var completion = CommandExtensions.CompleteAsync(_describeOnly, _withSync, decoder); if (completion.IsCompletedSuccessfully) { _completeError = completion.Result; @@ -270,7 +271,7 @@ async ValueTask DrainRowsAndComplete(PgDecoder decoder) if (message.Header.Type is not PgTypes.BackendType.DataRow) break; } - _completeError = await Command.CompleteAsync(decoder).ConfigureAwait(false); + _completeError = await CommandExtensions.CompleteAsync(_describeOnly, _withSync, decoder).ConfigureAwait(false); } catch (Exception ex) { @@ -280,10 +281,10 @@ async ValueTask DrainRowsAndComplete(PgDecoder decoder) } } - public void Initialize(CommandFlow flow, PgDecoder decoder) + public void Initialize(in Command command, PgDecoder decoder) { - if (!ReferenceEquals(_flow, flow)) - _flow = flow; + _describeOnly = command.DescribeOnly; + _withSync = command.WithSync; if (!ReferenceEquals(_decoder, decoder)) _decoder = decoder; @@ -292,13 +293,14 @@ public void Initialize(CommandFlow flow, PgDecoder decoder) _completeError = null; // A command is immediately done if we haven't submitted an execute. - _done = Command.DescribeOnly; + _done = _describeOnly; _first = !_done; } public void Reset() { - _flow = null!; + _describeOnly = false; + _withSync = false; _decoder = null!; _exceptionDispatchInfo = null; _completeError = null; diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 66a4870..085a7d0 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -583,7 +583,7 @@ await _commands.ItemRef(_commandIndex) CommandResult result; { ref readonly var readState = ref context.GetProtocolStatic(); - readState.ResultMessageEnumerator.Initialize(this, _decoder); + readState.ResultMessageEnumerator.Initialize(_commands.ItemRef(_commandIndex), _decoder); result = _enumeratorCurrent ?? readState.CommandResult; ref readonly var resultCommand = ref _commands.ItemRef(_commandIndex); From 8d809b3616ba399564fd94211d0ff91356941d4d Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 28 Aug 2026 17:01:10 +0200 Subject: [PATCH 002/136] Arm per-read cancellation after command dispatch --- Slon.Tests/Pg/CommandUserCancellationTests.cs | 33 ++++++++++++++ .../Protocol/Flows/CommandFlow.Enumerator.cs | 2 +- Slon/Pg/Protocol/Flows/CommandFlow.cs | 44 ++++++++++++++++--- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/Slon.Tests/Pg/CommandUserCancellationTests.cs b/Slon.Tests/Pg/CommandUserCancellationTests.cs index 3860186..d7970dc 100644 --- a/Slon.Tests/Pg/CommandUserCancellationTests.cs +++ b/Slon.Tests/Pg/CommandUserCancellationTests.cs @@ -128,6 +128,39 @@ public async Task UserCt_FiresMidRead_SurfacesOce_ProtocolUsable() await PgTestPool.RunAsync(protocol, "select 1"); } + // Autonomous execution may enter the command read before the consumer supplies its per-read + // token. The late token must still arm backend cancellation for that active read. + [TestMethod] + public async Task UserCt_SuppliedAfterReadStarted_RequestsCancellation_ProtocolUsable() + { + await using var blocker = await PgAdvisoryLock.AcquireAsync(); + var cancelRequested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var protocol = await PgTestPool.NewIsolatedAsync( + o => o.CancelSender = (processId, secretKey, token) => + { + cancelRequested.TrySetResult(); + return new(CancelRequestState.NotSent); + }); + + var flow = new CommandFlow(async: true, blocker.WaitCommand); + Assert.IsTrue(protocol.TryQueue(flow)); + await blocker.WaitUntilContendedAsync(protocol.FlowControl.BackendProcessId); + + using var cts = new CancellationTokenSource(); + var enumerator = flow.GetAsyncEnumerator(); + var moveNext = enumerator.MoveNextAsync(cts.Token).AsTask(); + cts.Cancel(); + + await cancelRequested.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await blocker.ReleaseAsync(); + var exception = await Assert.ThrowsExactlyAsync( + async () => await moveNext.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.AreEqual(cts.Token, exception.CancellationToken); + await enumerator.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + [TestMethod] public async Task UserCt_ThenProtocolClose_DuringDrain_CompletesPendingMoveNext() { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs index 7e2faf1..52d15af 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs @@ -258,7 +258,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken) // distinguish a pre-fired cancellation from a clean end. if (cancellationToken.CanBeCanceled) { - flow.GetOrCreateCancellationState().CallerToken = cancellationToken; + flow.SetCallerCancellationToken(cancellationToken); flow._enumeratorMoveNextTaskSource.CanCompleteConcurrently = true; } diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 085a7d0..9e14a38 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -623,6 +623,12 @@ await _commands.ItemRef(_commandIndex) if (!publishedResult && IsAsync) { await _callerInteractionCore.WaitForCaller(this).ConfigureAwait(false); + // The first consumer can arrive after the response prelude was already read + // and its registrations were disposed. Its token was armed by MoveNextAsync; + // the result has now won that race, so retire the late registration before + // publishing the result. + if (Volatile.Read(ref _cancellationState) is { } lateCancellation) + await DisposeCancellationRegistrations(lateCancellation).ConfigureAwait(false); EnterStoppingDrainIfNeeded(context); } @@ -883,15 +889,32 @@ static void ReadRfq(PgDecoder decoder) } } + void SetCallerCancellationToken(CancellationToken token) + { + var cancellation = GetOrCreateCancellationState(); + lock (cancellation) + { + cancellation.CallerToken = token; + RegisterCancellationCallbacksLocked(cancellation); + } + } + void RegisterCancellationCallbacks(CancellationState cancellation) + { + lock (cancellation) + RegisterCancellationCallbacksLocked(cancellation); + } + + void RegisterCancellationCallbacksLocked(CancellationState cancellation) { if (cancellation.CallerToken.CanBeCanceled) { Debug.Assert(IsAsync); - cancellation.CallerRegistration = cancellation.CallerToken.UnsafeRegister(static (state, token) - => ((CommandFlow)state!).RequestCancelAndWake(token, CancellationScope.CurrentWindow), this); + if (cancellation.CallerRegistration == default) + cancellation.CallerRegistration = cancellation.CallerToken.UnsafeRegister(static (state, token) + => ((CommandFlow)state!).RequestCancelAndWake(token, CancellationScope.CurrentWindow), this); } - if (cancellation.FlowToken.CanBeCanceled) + if (cancellation.FlowToken.CanBeCanceled && cancellation.FlowRegistration == default) { cancellation.FlowRegistration = cancellation.FlowToken.UnsafeRegister(static (state, token) => ((CommandFlow)state!).RequestCancelAndWake(token, CancellationScope.RemainingFlow), this); @@ -901,10 +924,17 @@ void RegisterCancellationCallbacks(CancellationState cancellation) [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask DisposeCancellationRegistrations(CancellationState cancellation) { - if (cancellation.CallerToken.CanBeCanceled) - await cancellation.CallerRegistration.DisposeAsync().ConfigureAwait(false); - if (cancellation.FlowToken.CanBeCanceled) - await cancellation.FlowRegistration.DisposeAsync().ConfigureAwait(false); + CancellationTokenRegistration callerRegistration; + CancellationTokenRegistration flowRegistration; + lock (cancellation) + { + callerRegistration = cancellation.CallerRegistration; + cancellation.CallerRegistration = default; + flowRegistration = cancellation.FlowRegistration; + cancellation.FlowRegistration = default; + } + await callerRegistration.DisposeAsync().ConfigureAwait(false); + await flowRegistration.DisposeAsync().ConfigureAwait(false); } bool IsCancellationToken(CancellationToken token) From bfd5ad75bfc84f70f2966ea6f7958284ffce0e24 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 19:27:36 +0200 Subject: [PATCH 003/136] Reduce backend message publication state --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 15 ++ Slon/Pg/Protocol/BackendMessage.cs | 193 ++++++++++++------ Slon/Pg/Protocol/BackendMessageContext.cs | 30 ++- 3 files changed, 171 insertions(+), 67 deletions(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index 93e9620..09b2841 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -159,6 +159,21 @@ public void BackendMessage_BodyAccessThrowsAfterStreamingWindowAdvances() Assert.ThrowsExactly(() => context.TryExtend(0, out _)); } + [TestMethod] + public void IndependentBackendMessage_ReconstructsAChainedBuffer() + { + var bytes = BackendMessageBytes(BackendType.DataRow, [1, 2, 3, 4, 5, 6]); + var sequence = Segmented(bytes.AsMemory(0, 7), bytes.AsMemory(7)); + var message = BackendMessage.CreateIndependent( + new BackendHeader(BackendType.DataRow, bytes.Length - 1), sequence); + + CollectionAssert.AreEqual(bytes.AsSpan(BackendHeader.ByteCount).ToArray(), + message.GetSequence().ToArray()); + Assert.IsTrue(message.TryGetFirstSpan(0, out var first)); + CollectionAssert.AreEqual(bytes.AsSpan(BackendHeader.ByteCount, 2).ToArray(), + first.ToArray()); + } + [TestMethod] public void BackendMessageContext_CurrentThrowsOutsidePublicationWindow() { diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index ac6a3f2..5384ce5 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -13,28 +13,54 @@ public readonly struct BackendMessage { string DebuggerDisplay => $"Type = {Header.Type}, Length = {Header.MessageLength}"; - readonly ReadOnlySequence _buffer; - readonly BackendMessageContext _context; + readonly object? _firstObject; + readonly object? _contextOrEndObject; - // Packed to avoid another 8 bytes. - readonly bool _buffered; - readonly BackendType _type; - readonly short _token; + // Buffered, peeked, type, and token fit in one word so the message remains 32 bytes. + readonly uint _state; readonly int _length; + readonly int _startIndex; + readonly int _endIndexOrBufferedLength; - BackendMessage(BackendHeader header, ReadOnlySequence buffer, BackendMessageContext context, short token, bool buffered) + BackendMessage(BackendHeader header, ReadOnlySequence buffer, + BackendMessageContext? context, short token, bool buffered, + bool peeked = false, bool independent = false) { - _buffer = buffer; - _context = context; - _buffered = buffered; - _type = header.Type; - _token = token; + _firstObject = buffer.Start.GetObject(); + _contextOrEndObject = independent ? buffer.End.GetObject() : context; + _startIndex = buffer.Start.GetInteger() & int.MaxValue; + _endIndexOrBufferedLength = independent + ? buffer.End.GetInteger() & int.MaxValue + : checked((int)buffer.Length); + _state = (buffered ? 1u : 0) + | (peeked ? 2u : 0) + | (independent ? 4u : 0) + | ((uint)(byte)header.Type << 3) + | ((uint)(ushort)token << 11); _length = header.Length; + if (context is not null && !peeked) + context.SetCurrentFallbackBuffer(in buffer, + _firstObject is ReadOnlySequenceSegment); } internal BackendMessage(BackendHeader header, ReadOnlySequence buffer, BackendMessageContext context, short token) : this(header, buffer, context, token, buffer.Length >= header.MessageLength) {} + internal static BackendMessage CreatePeeked(BackendHeader header, + ReadOnlySequence buffer, BackendMessageContext context, short token) + => new(header, buffer, context, token, + buffer.Length >= header.MessageLength, peeked: true); + + internal static BackendMessage CreateIndependent( + BackendHeader header, ReadOnlySequence buffer) + { + if (buffer.Length < header.MessageLength) + ThrowHelper.ThrowInvalidOperation( + "An independent backend message must be fully buffered."); + return new(header, buffer, context: null, token: 0, + buffered: true, independent: true); + } + internal static void Initialize(ref BackendMessage destination, BackendHeader header, ReadOnlySequence buffer, BackendMessageContext context, short token, bool buffered) { @@ -47,44 +73,83 @@ internal static void Initialize(ref BackendMessage destination, BackendHeader he [MethodImpl(MethodImplOptions.AggressiveInlining)] static void WriteGranularly(ref BackendMessage destination, in BackendMessage value, bool destinationIsZero = false) { - if ((destinationIsZero && value._context is not null) || !ReferenceEquals(destination._context, value._context)) - Unsafe.AsRef(in destination._context) = value._context!; + if ((destinationIsZero && value._contextOrEndObject is not null) + || !ReferenceEquals(destination._contextOrEndObject, value._contextOrEndObject)) + Unsafe.AsRef(in destination._contextOrEndObject) = value._contextOrEndObject; + if (!ReferenceEquals(destination._firstObject, value._firstObject)) + Unsafe.AsRef(in destination._firstObject) = value._firstObject; - WriteGranularly(ref Unsafe.AsRef(in destination._buffer), in value._buffer); - - Unsafe.AsRef(in destination._buffered) = value._buffered; - Unsafe.AsRef(in destination._type) = value._type; - Unsafe.AsRef(in destination._token) = value._token; + Unsafe.AsRef(in destination._state) = value._state; Unsafe.AsRef(in destination._length) = value._length; + Unsafe.AsRef(in destination._startIndex) = value._startIndex; + Unsafe.AsRef(in destination._endIndexOrBufferedLength) = value._endIndexOrBufferedLength; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void WriteGranularly(ref ReadOnlySequence destination, in ReadOnlySequence value) - => destination = value; + BackendType Type => (BackendType)((_state >> 3) & byte.MaxValue); + short Token => (short)(_state >> 11); + bool IsPeeked => (_state & 2) != 0; + bool IsIndependent => (_state & 4) != 0; + internal bool IsDefault => Type == default; + BackendMessageContext Context + => _contextOrEndObject as BackendMessageContext + ?? throw new InvalidOperationException( + "The independent backend message has no decoder context."); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void SetSequence(ref ReadOnlySequence destination, in ReadOnlySequence value) - => WriteGranularly(ref destination, in value); + ReadOnlySequence GetBuffer() + { + if (!IsIndependent) + { + if (_firstObject is byte[] array) + return new(array, _startIndex, _endIndexOrBufferedLength); + if (_firstObject is MemoryManager manager) + return new(manager.Memory.Slice(_startIndex, _endIndexOrBufferedLength)); + return Context.GetFallbackBuffer(Token, IsPeeked); + } - BackendType Type => _type; - internal bool IsDefault => _type == default; + if (_firstObject is byte[] independentArray + && ReferenceEquals(_firstObject, _contextOrEndObject)) + return new(independentArray, _startIndex, + _endIndexOrBufferedLength - _startIndex); + if (_firstObject is MemoryManager independentManager + && ReferenceEquals(_firstObject, _contextOrEndObject)) + return new(independentManager.Memory.Slice( + _startIndex, _endIndexOrBufferedLength - _startIndex)); + return new((ReadOnlySequenceSegment)_firstObject!, _startIndex, + (ReadOnlySequenceSegment)_contextOrEndObject!, + _endIndexOrBufferedLength); + } public BackendHeader Header { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => BackendHeader.CreateUnchecked(_type, _length); + get => BackendHeader.CreateUnchecked(Type, _length); } public ReadOnlySequence GetSequence(SequencePosition start) { EnsureBodyWindowAvailable(); - return _buffer.Slice(start); + return GetBuffer().Slice(start); } public ReadOnlySequence GetSequence(long offset) { EnsureBodyWindowAvailable(); - return _buffer.Slice(BackendHeader.ByteCount + offset); + ArgumentOutOfRangeException.ThrowIfNegative(offset); + var start = checked(BackendHeader.ByteCount + offset); + var length = BufferedLength - start; + if (length < 0 || length > int.MaxValue) + throw new ArgumentOutOfRangeException(nameof(offset)); + + if (!IsIndependent || ReferenceEquals(_firstObject, _contextOrEndObject)) + { + if (_firstObject is byte[] array) + return new(array, checked(_startIndex + (int)start), (int)length); + if (_firstObject is MemoryManager manager) + return new(manager.Memory.Slice( + checked(_startIndex + (int)start), (int)length)); + } + return GetBuffer().Slice(start); } public ReadOnlySequence GetSequence() @@ -101,17 +166,7 @@ public bool TryGetFirstSpan(int offset, out ReadOnlySpan span) internal bool TryGetFirstSpanUnchecked(int offset, out ReadOnlySpan span) { offset += BackendHeader.ByteCount; - ref var buffer = ref Unsafe.AsRef(in _buffer); - ReadOnlySpan firstSpan; - if (SequenceMarshal.TryGetArray(buffer, out var array)) - { - Debug.Assert(buffer.IsSingleSegment); - firstSpan = array.AsSpan(); - } - else - { - firstSpan = buffer.FirstSpan; - } + var firstSpan = GetFirstMemory().Span; if ((uint)offset <= (uint)firstSpan.Length) { span = firstSpan.Slice(offset); @@ -125,19 +180,9 @@ internal bool TryGetFirstSpanUnchecked(int offset, out ReadOnlySpan span) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory memory) { - Debug.Assert(_buffered); + Debug.Assert(Buffered); offset += BackendHeader.ByteCount; - ref var buffer = ref Unsafe.AsRef(in _buffer); - ReadOnlyMemory firstMemory; - if (SequenceMarshal.TryGetArray(buffer, out var array)) - { - Debug.Assert(buffer.IsSingleSegment); - firstMemory = array.AsMemory(); - } - else - { - firstMemory = buffer.First; - } + var firstMemory = GetFirstMemory(); if ((uint)offset <= (uint)firstMemory.Length) { memory = firstMemory.Slice(offset); @@ -148,13 +193,28 @@ internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory mem return false; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + ReadOnlyMemory GetFirstMemory() + { + var length = IsIndependent + ? _endIndexOrBufferedLength - _startIndex + : _endIndexOrBufferedLength; + return _firstObject switch + { + byte[] array => array.AsMemory(_startIndex, length), + MemoryManager manager => manager.Memory.Slice(_startIndex, length), + ReadOnlySequenceSegment segment => segment.Memory.Slice(_startIndex), + _ => GetBuffer().First + }; + } + public SequenceReader BodyReader => new(GetSequence()); [MethodImpl(MethodImplOptions.AggressiveInlining)] void EnsureBodyWindowAvailable() { - if (!_buffered) - _context.EnsureBodyWindowAvailable(_token); + if (!Buffered) + Context.EnsureBodyWindowAvailable(Token); } public (PgError? Error, BackendType Type) EnsureExpectedOrError(params ReadOnlySpan expected) @@ -181,19 +241,19 @@ static void Throw(BackendType actual, ReadOnlySpan expected) $"Unexpected backend message: {actual}, expected: {string.Join(" or ", expected.ToArray())}."); } - public Accessor GetAccessor() => new(_context, _token); + public Accessor GetAccessor() => new(Context, Token); internal BackendMessageBodyReader OpenBodyReader() - => new(_context, _token, GetSequence(), Buffered); + => new(Context, Token, GetSequence(), Buffered); internal void BufferBody() { if (!Buffered) - _context.BufferCurrentMessage(_token); + Context.BufferCurrentMessage(Token); } internal ValueTask BufferBodyAsync(CancellationToken cancellationToken) - => Buffered ? default : _context.BufferCurrentMessageAsync(_token, cancellationToken); + => Buffered ? default : Context.BufferCurrentMessageAsync(Token, cancellationToken); public readonly struct Accessor { @@ -325,23 +385,26 @@ PgError CreateError(ReadOnlySpan expected, bool unhandled = true) } internal void MarkPriorCancellationExposure() - => _context.MarkPriorCancellationExposure(_token); + => Context.MarkPriorCancellationExposure(Token); internal bool HasPriorCancellationExposure - => _context.HasPriorCancellationExposure(_token); + => Context.HasPriorCancellationExposure(Token); internal void MarkBackendTermination() - => _context.MarkBackendTermination(_token); + => Context.MarkBackendTermination(Token); internal bool IsBackendTermination - => _context.IsBackendTermination(_token); + => Context.IsBackendTermination(Token); internal bool TryObserveError() - => _context.TryObserveError(_token); + => Context.TryObserveError(Token); // We have no buffer for header only messages. - public bool Buffered => _buffered; - internal long BufferedLength => _buffer.Length; + public bool Buffered => (_state & 1) != 0; + internal long BufferedLength + => IsDefault ? 0 + : IsIndependent ? Header.MessageLength + : _endIndexOrBufferedLength; } [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 43fde2c..6b36afe 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -11,6 +11,7 @@ sealed class BackendMessageContext PgDecoder _decoder = null!; BackendMessageBatch _remainingBatch; BackendMessage _current; + ReadOnlySequence _currentFallbackBuffer; short _version; const byte PriorCancellationExposure = 1 << 0; const byte BackendTermination = 1 << 1; @@ -60,6 +61,29 @@ public BackendMessage GetCurrent(short token) return _current; } + internal void SetCurrentFallbackBuffer( + in ReadOnlySequence buffer, bool required) + { + if (required) + _currentFallbackBuffer = buffer; + else if (!_currentFallbackBuffer.IsEmpty) + _currentFallbackBuffer = default; + } + + internal ReadOnlySequence GetFallbackBuffer(short token, bool peeked) + { + if (peeked) + { + if (!_hasPeeked || _version != token) + ThrowHelper.ThrowInvalidOperation( + "Backend message has been invalidated by moving to the next message."); + return _peekedBuffer; + } + + Validate(token); + return _currentFallbackBuffer; + } + public long GetCurrentMessageOffset(short token) { Validate(token); @@ -251,6 +275,7 @@ public void RetireCurrentBatch() // A failed message poll preserves Current, but crossing this ownership boundary cannot. var invalidateToken = !_current.IsDefault || _hasPeeked; _current = default; + _currentFallbackBuffer = default; _hasPeeked = false; _peekedHeader = default; _peekedBuffer = default; @@ -289,7 +314,7 @@ public bool TryPeekNext(out BackendHeader header) header = default; return false; } - BackendMessage.SetSequence(ref _peekedBuffer, in buffer); + _peekedBuffer = buffer; _hasPeeked = true; header = _peekedHeader; return true; @@ -300,7 +325,8 @@ public BackendMessage Peeked get { Debug.Assert(_hasPeeked); - return new(_peekedHeader, _peekedBuffer, this, _version); + return BackendMessage.CreatePeeked( + _peekedHeader, _peekedBuffer, this, _version); } } From b96947a3d4fa043c055ca914bb73c9246272ed22 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 19:31:35 +0200 Subject: [PATCH 004/136] Publish peeked backend messages directly --- Slon/Pg/Protocol/BackendMessage.cs | 36 +++++++++++++---------- Slon/Pg/Protocol/BackendMessageContext.cs | 32 ++++++-------------- 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index 5384ce5..3bf1c6e 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -24,7 +24,7 @@ public readonly struct BackendMessage BackendMessage(BackendHeader header, ReadOnlySequence buffer, BackendMessageContext? context, short token, bool buffered, - bool peeked = false, bool independent = false) + bool independent = false) { _firstObject = buffer.Start.GetObject(); _contextOrEndObject = independent ? buffer.End.GetObject() : context; @@ -33,12 +33,11 @@ public readonly struct BackendMessage ? buffer.End.GetInteger() & int.MaxValue : checked((int)buffer.Length); _state = (buffered ? 1u : 0) - | (peeked ? 2u : 0) - | (independent ? 4u : 0) - | ((uint)(byte)header.Type << 3) - | ((uint)(ushort)token << 11); + | (independent ? 2u : 0) + | ((uint)(byte)header.Type << 2) + | ((uint)(ushort)token << 10); _length = header.Length; - if (context is not null && !peeked) + if (context is not null) context.SetCurrentFallbackBuffer(in buffer, _firstObject is ReadOnlySequenceSegment); } @@ -46,11 +45,6 @@ public readonly struct BackendMessage internal BackendMessage(BackendHeader header, ReadOnlySequence buffer, BackendMessageContext context, short token) : this(header, buffer, context, token, buffer.Length >= header.MessageLength) {} - internal static BackendMessage CreatePeeked(BackendHeader header, - ReadOnlySequence buffer, BackendMessageContext context, short token) - => new(header, buffer, context, token, - buffer.Length >= header.MessageLength, peeked: true); - internal static BackendMessage CreateIndependent( BackendHeader header, ReadOnlySequence buffer) { @@ -61,6 +55,17 @@ internal static BackendMessage CreateIndependent( buffered: true, independent: true); } + internal static void InitializeIndependent(ref BackendMessage destination, + BackendHeader header, ReadOnlySequence buffer) + { + var value = CreateIndependent(header, buffer); + WriteGranularly(ref destination, in value); + } + + internal static void Copy( + ref BackendMessage destination, in BackendMessage value) + => WriteGranularly(ref destination, in value); + internal static void Initialize(ref BackendMessage destination, BackendHeader header, ReadOnlySequence buffer, BackendMessageContext context, short token, bool buffered) { @@ -85,10 +90,9 @@ static void WriteGranularly(ref BackendMessage destination, in BackendMessage va Unsafe.AsRef(in destination._endIndexOrBufferedLength) = value._endIndexOrBufferedLength; } - BackendType Type => (BackendType)((_state >> 3) & byte.MaxValue); - short Token => (short)(_state >> 11); - bool IsPeeked => (_state & 2) != 0; - bool IsIndependent => (_state & 4) != 0; + BackendType Type => (BackendType)((_state >> 2) & byte.MaxValue); + short Token => (short)(_state >> 10); + bool IsIndependent => (_state & 2) != 0; internal bool IsDefault => Type == default; BackendMessageContext Context => _contextOrEndObject as BackendMessageContext @@ -104,7 +108,7 @@ ReadOnlySequence GetBuffer() return new(array, _startIndex, _endIndexOrBufferedLength); if (_firstObject is MemoryManager manager) return new(manager.Memory.Slice(_startIndex, _endIndexOrBufferedLength)); - return Context.GetFallbackBuffer(Token, IsPeeked); + return Context.GetFallbackBuffer(Token); } if (_firstObject is byte[] independentArray diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 6b36afe..a89dcbc 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -25,8 +25,7 @@ sealed class BackendMessageContext // alone owns validity; leaving the inactive buffer populated avoids a redundant clear and lets // the next peek usually reuse the same backing objects without write barriers. bool _hasPeeked; - BackendHeader _peekedHeader; - ReadOnlySequence _peekedBuffer; + BackendMessage _peeked; long _currentMessageOffset; public BackendMessage Current @@ -70,16 +69,8 @@ internal void SetCurrentFallbackBuffer( _currentFallbackBuffer = default; } - internal ReadOnlySequence GetFallbackBuffer(short token, bool peeked) + internal ReadOnlySequence GetFallbackBuffer(short token) { - if (peeked) - { - if (!_hasPeeked || _version != token) - ThrowHelper.ThrowInvalidOperation( - "Backend message has been invalidated by moving to the next message."); - return _peekedBuffer; - } - Validate(token); return _currentFallbackBuffer; } @@ -252,8 +243,8 @@ public bool TryMoveNext() { _hasPeeked = false; ResetMessageState(); - BackendMessage.Initialize(ref _current, _peekedHeader, _peekedBuffer, this, ++_version, - _peekedBuffer.Length >= _peekedHeader.MessageLength); + _version++; + BackendMessage.Copy(ref _current, in _peeked); return true; } if (!_remainingBatch.TryReadNextInPlace(out var header, out var buffer, out var bufferLength)) @@ -277,8 +268,6 @@ public void RetireCurrentBatch() _current = default; _currentFallbackBuffer = default; _hasPeeked = false; - _peekedHeader = default; - _peekedBuffer = default; _remainingBatch = default; _currentMessageOffset = 0; _messageState = 0; @@ -296,7 +285,7 @@ public bool TryPeekNextType(out PgTypes.BackendType type) { if (_hasPeeked) { - type = _peekedHeader.Type; + type = _peeked.Header.Type; return true; } return _remainingBatch.TryPeekType(out type); @@ -306,17 +295,15 @@ public bool TryPeekNext(out BackendHeader header) { if (_hasPeeked) { - header = _peekedHeader; + header = _peeked.Header; return true; } - if (!_remainingBatch.TryReadNextInPlace(out _peekedHeader, out var buffer, out _)) + if (!_remainingBatch.TryReadNextInPlace(out header, out var buffer, out _)) { - header = default; return false; } - _peekedBuffer = buffer; + BackendMessage.InitializeIndependent(ref _peeked, header, buffer); _hasPeeked = true; - header = _peekedHeader; return true; } @@ -325,8 +312,7 @@ public BackendMessage Peeked get { Debug.Assert(_hasPeeked); - return BackendMessage.CreatePeeked( - _peekedHeader, _peekedBuffer, this, _version); + return _peeked; } } From 330e96ae7829d3d9be9b4d491bcaf4b0beee65de Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 19:42:31 +0200 Subject: [PATCH 005/136] Use one backend message publication slot --- Slon/Pg/Protocol/BackendMessageContext.cs | 61 +++++++++++------------ 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index a89dcbc..b805fdc 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -20,12 +20,8 @@ sealed class BackendMessageContext const byte MessageOffsetCaptured = 1 << 4; byte _messageState; - // Peek slot: TryPeekNext advances the real batch cursor into here, so the header parse - // happens at peek time and a follow-up TryMoveNext can publish without re-parsing. _hasPeeked - // alone owns validity; leaving the inactive buffer populated avoids a redundant clear and lets - // the next peek usually reuse the same backing objects without write barriers. - bool _hasPeeked; - BackendMessage _peeked; + enum PublicationState : byte { None, Current, Peeked } + PublicationState _publicationState; long _currentMessageOffset; public BackendMessage Current @@ -34,7 +30,7 @@ public BackendMessage Current get { var current = _current; - if (current.IsDefault) + if (_publicationState is not PublicationState.Current) ThrowHelper.ThrowInvalidOperation("The decoder has no current backend message."); return current; } @@ -43,19 +39,23 @@ public BackendMessage Current public bool CurrentIsError { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => _current.Header.Type is PgTypes.BackendType.ErrorResponse; + get + { + Debug.Assert(_publicationState is PublicationState.Current); + return _current.Header.Type is PgTypes.BackendType.ErrorResponse; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGetCurrent(out BackendMessage current) { current = _current; - return !current.IsDefault; + return _publicationState is PublicationState.Current; } public BackendMessage GetCurrent(short token) { - if (_version != token) + if (_publicationState is not PublicationState.Current || _version != token) ThrowHelper.ThrowInvalidOperation("Backend message has been invalidated by moving to the next message."); return _current; } @@ -82,7 +82,7 @@ public long GetCurrentMessageOffset(short token) { // Fully buffered messages never need their batch-relative offset. Capture it only // before a streaming operation can replace the segment used to derive it. - Debug.Assert(!_current.Buffered && !_hasPeeked); + Debug.Assert(!_current.Buffered); _currentMessageOffset = _remainingBatch.ConsumedLength - _current.BufferedLength; _messageState |= MessageOffsetCaptured; } @@ -198,7 +198,7 @@ void SetCurrentFromSegment(short token, ReadOnlySequence segment) void Validate(short token) { - if (_version != token) + if (_publicationState is PublicationState.Peeked || _version != token) ThrowHelper.ThrowInvalidOperation("Backend message has been invalidated by moving to the next message."); } @@ -239,12 +239,9 @@ public bool TryObserveError(short token) public bool TryMoveNext() { - if (_hasPeeked) + if (_publicationState is PublicationState.Peeked) { - _hasPeeked = false; - ResetMessageState(); - _version++; - BackendMessage.Copy(ref _current, in _peeked); + _publicationState = PublicationState.Current; return true; } if (!_remainingBatch.TryReadNextInPlace(out var header, out var buffer, out var bufferLength)) @@ -252,6 +249,7 @@ public bool TryMoveNext() ResetMessageState(); BackendMessage.Initialize(ref _current, header, buffer, this, ++_version, bufferLength >= header.MessageLength); + _publicationState = PublicationState.Current; return true; void ResetMessageState() @@ -264,10 +262,10 @@ public void RetireCurrentBatch() { // Moving the batch enumerator may return or refill the memory backing every view held here. // A failed message poll preserves Current, but crossing this ownership boundary cannot. - var invalidateToken = !_current.IsDefault || _hasPeeked; + var invalidateToken = _publicationState is not PublicationState.None; _current = default; _currentFallbackBuffer = default; - _hasPeeked = false; + _publicationState = PublicationState.None; _remainingBatch = default; _currentMessageOffset = 0; _messageState = 0; @@ -283,9 +281,9 @@ public void RetireCurrentBatch() [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryPeekNextType(out PgTypes.BackendType type) { - if (_hasPeeked) + if (_publicationState is PublicationState.Peeked) { - type = _peeked.Header.Type; + type = _current.Header.Type; return true; } return _remainingBatch.TryPeekType(out type); @@ -293,17 +291,20 @@ public bool TryPeekNextType(out PgTypes.BackendType type) public bool TryPeekNext(out BackendHeader header) { - if (_hasPeeked) + if (_publicationState is PublicationState.Peeked) { - header = _peeked.Header; + header = _current.Header; return true; } if (!_remainingBatch.TryReadNextInPlace(out header, out var buffer, out _)) { return false; } - BackendMessage.InitializeIndependent(ref _peeked, header, buffer); - _hasPeeked = true; + _version++; + _messageState = 0; + _currentFallbackBuffer = default; + BackendMessage.InitializeIndependent(ref _current, header, buffer); + _publicationState = PublicationState.Peeked; return true; } @@ -311,19 +312,17 @@ public BackendMessage Peeked { get { - Debug.Assert(_hasPeeked); - return _peeked; + Debug.Assert(_publicationState is PublicationState.Peeked); + return _current; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void SetBatch(BackendMessageBatch batch) { - Debug.Assert(_current.IsDefault && !_hasPeeked, + Debug.Assert(_publicationState is PublicationState.None, "The prior batch must be retired before publishing replacement storage."); - // Keep release behavior defensive. The inactive buffer may stay populated because - // _hasPeeked owns validity and the next peek overwrites it. - _hasPeeked = false; + _publicationState = PublicationState.None; _remainingBatch = batch; } From 8dc4f7c8a8ad06d3a05bea3dd6089b685688c674 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 20:04:11 +0200 Subject: [PATCH 006/136] Parse backend messages once before publication --- Slon/Pg/Protocol/BackendMessageBatch.cs | 25 ----------------------- Slon/Pg/Protocol/BackendMessageContext.cs | 19 ++++------------- Slon/Pg/Protocol/PgDecoder.cs | 5 ++--- Slon/Pg/Protocol/ProtocolReadPipe.cs | 1 - 4 files changed, 6 insertions(+), 44 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageBatch.cs b/Slon/Pg/Protocol/BackendMessageBatch.cs index da1742a..eb7050b 100644 --- a/Slon/Pg/Protocol/BackendMessageBatch.cs +++ b/Slon/Pg/Protocol/BackendMessageBatch.cs @@ -18,31 +18,6 @@ struct BackendMessageBatch(ReadOnlySequence buffer) public readonly long ConsumedLength => _consumedLength; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryPeekType(out BackendType type) - { - var span = _buffer.FirstSpan; - if (!span.IsEmpty) - { - type = (BackendType)span[0]; - return true; - } - return TryPeekTypeMultiSegment(_buffer.Sequence, out type); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - static bool TryPeekTypeMultiSegment(ReadOnlySequence buffer, out BackendType type) - { - var reader = new SequenceReader(buffer); - if (reader.TryPeek(out var tag)) - { - type = (BackendType)tag; - return true; - } - type = default; - return false; - } - public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) { if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) && !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader)) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index b805fdc..96622ea 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -278,17 +278,6 @@ public void RetireCurrentBatch() // slot and the follow-up TryMoveNext picks it up without re-parsing. The returned // BackendMessage is valid until the next TryMoveNext (which bumps the version token); // use it immediately, don't store it. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryPeekNextType(out PgTypes.BackendType type) - { - if (_publicationState is PublicationState.Peeked) - { - type = _current.Header.Type; - return true; - } - return _remainingBatch.TryPeekType(out type); - } - public bool TryPeekNext(out BackendHeader header) { if (_publicationState is PublicationState.Peeked) @@ -296,14 +285,14 @@ public bool TryPeekNext(out BackendHeader header) header = _current.Header; return true; } - if (!_remainingBatch.TryReadNextInPlace(out header, out var buffer, out _)) + if (!_remainingBatch.TryReadNextInPlace( + out header, out var buffer, out var bufferLength)) { return false; } - _version++; _messageState = 0; - _currentFallbackBuffer = default; - BackendMessage.InitializeIndependent(ref _current, header, buffer); + BackendMessage.Initialize(ref _current, header, buffer, this, ++_version, + bufferLength >= header.MessageLength); _publicationState = PublicationState.Peeked; return true; } diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 2621378..4b46abb 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -689,8 +689,9 @@ bool TryMoveNextCore() { while (true) { - while (_pipe.TryPeekNextType(out var type)) + while (_pipe.TryPeekNext(out var header)) { + var type = header.Type; // Only auto-handled messages need the transactional peek slot: their handler may need // I/O and decline the synchronous path. Ordinary messages can publish directly. if (type is not (PgTypes.BackendType.ReadyForQuery @@ -703,8 +704,6 @@ or PgTypes.BackendType.NotificationResponse return true; } - if (!_pipe.TryPeekNext(out _)) - break; var handled = false; if (type is PgTypes.BackendType.ReadyForQuery) RestoreDefaultReadTimeout(); diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index 29e7bc0..140dea1 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -18,7 +18,6 @@ sealed class ProtocolReadPipe(PipeSegmentEnumerator _messageContext.TryGetCurrent(out message); public bool TryMoveNext() => _messageContext.TryMoveNext(); - public bool TryPeekNextType(out PgTypes.BackendType type) => _messageContext.TryPeekNextType(out type); public bool TryPeekNext(out BackendHeader header) => _messageContext.TryPeekNext(out header); public BackendMessage Peeked => _messageContext.Peeked; From f285493a9820e697dbf9e6c65770da5cda4a0c87 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 20:26:08 +0200 Subject: [PATCH 007/136] Derive backend message offsets on demand --- Slon/Pg/Protocol/BackendMessageBatch.cs | 16 ++++++++++------ Slon/Pg/Protocol/BackendMessageContext.cs | 6 ++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageBatch.cs b/Slon/Pg/Protocol/BackendMessageBatch.cs index eb7050b..79e7b6c 100644 --- a/Slon/Pg/Protocol/BackendMessageBatch.cs +++ b/Slon/Pg/Protocol/BackendMessageBatch.cs @@ -11,12 +11,17 @@ namespace Slon.Pg.Protocol; struct BackendMessageBatch(ReadOnlySequence buffer) { FastReadOnlySequence _buffer = new(buffer); - long _consumedLength; + long _initialLength = buffer.Length; - BackendMessageBatch(ReadOnlySequence buffer, long consumedLength) : this(buffer) - => _consumedLength = consumedLength; + public readonly long GetCurrentMessageOffset(long currentBufferedLength) + => _initialLength - _buffer.Length - currentBufferedLength; - public readonly long ConsumedLength => _consumedLength; + public readonly BackendMessageBatch Slice(long offset) + { + var result = new BackendMessageBatch(_buffer.Sequence.Slice(offset)); + result._initialLength = _initialLength; + return result; + } public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) { @@ -31,7 +36,6 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence Date: Tue, 1 Sep 2026 20:45:06 +0200 Subject: [PATCH 008/136] Track backend batch sequence positions directly --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 16 +++ Slon/Pg/Protocol/BackendMessageBatch.cs | 124 +++++++++++++----- 2 files changed, 104 insertions(+), 36 deletions(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index 09b2841..4ee073a 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -198,6 +198,22 @@ public void BackendMessageContext_CurrentThrowsOutsidePublicationWindow() Assert.ThrowsExactly(() => _ = accessor.Message); } + [TestMethod] + public void BackendMessageBatch_AdvancesAcrossExactSegmentBoundary() + { + var first = BackendMessageBytes(BackendType.CommandComplete, 6); + var second = BackendMessageBytes(BackendType.ReadyForQuery, 6); + var batch = new BackendMessageBatch(Segmented(first, second)); + + Assert.IsTrue(batch.TryReadNextInPlace(out var header, out var message, out _)); + Assert.AreEqual(BackendType.CommandComplete, header.Type); + CollectionAssert.AreEqual(first, message.ToArray()); + Assert.IsTrue(batch.TryReadNextInPlace(out header, out message, out _)); + Assert.AreEqual(BackendType.ReadyForQuery, header.Type); + CollectionAssert.AreEqual(second, message.ToArray()); + Assert.IsFalse(batch.TryReadNextInPlace(out _, out _, out _)); + } + [TestMethod] public async Task MovingToNextBatch_RetiresCurrentBeforeReturningItsStorage() { diff --git a/Slon/Pg/Protocol/BackendMessageBatch.cs b/Slon/Pg/Protocol/BackendMessageBatch.cs index 79e7b6c..c1be909 100644 --- a/Slon/Pg/Protocol/BackendMessageBatch.cs +++ b/Slon/Pg/Protocol/BackendMessageBatch.cs @@ -1,7 +1,6 @@ using System.Buffers; using System.Diagnostics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using Slon.Pipelines; using static Slon.Pg.Protocol.PgTypes; @@ -130,69 +129,122 @@ public OperationStatus CreateSegment(in ReadOnlySequence buffer, out long }; } - // TODO faster firstspan and splitting should be able to be upstreamed. - // Optimizes for faster splitting and length checks. + // Keeps the public position components scalar so consuming the first segment does not repeatedly + // reconstruct and rediscover the backing of a ReadOnlySequence. Materialize one only at API seams. struct FastReadOnlySequence { - ReadOnlySequence _sequence; + object? _startObject; + object? _endObject; + int _startIndex; + int _endIndex; long _length; - FastReadOnlySequence(ReadOnlySequence sequence, long length) + FastReadOnlySequence(object? startObject, int startIndex, + object? endObject, int endIndex, long length) { Debug.Assert(Unsafe.SizeOf>() is 32); - _sequence = sequence; + _startObject = startObject; + _endObject = endObject; + _startIndex = startIndex; + _endIndex = endIndex; _length = length; } public FastReadOnlySequence(ReadOnlySequence sequence) { Debug.Assert(Unsafe.SizeOf>() is 32); - _sequence = sequence; + _startObject = sequence.Start.GetObject(); + _endObject = sequence.End.GetObject(); + _startIndex = sequence.Start.GetInteger() & int.MaxValue; + _endIndex = sequence.End.GetInteger() & int.MaxValue; _length = sequence.Length; } - public ReadOnlySequence Sequence => _sequence; + public ReadOnlySequence Sequence + { + get + { + if (_startObject is null) + return default; + if (_startObject is T[] array) + { + Debug.Assert(ReferenceEquals(_startObject, _endObject)); + return new(array, _startIndex, _endIndex - _startIndex); + } + if (_startObject is MemoryManager manager) + { + Debug.Assert(ReferenceEquals(_startObject, _endObject)); + return new(manager.Memory.Slice( + _startIndex, _endIndex - _startIndex)); + } + return new((ReadOnlySequenceSegment)_startObject!, _startIndex, + (ReadOnlySequenceSegment)_endObject!, _endIndex); + } + } public long Length => _length; - public ReadOnlySpan FirstSpan => GetFirstSpan(out _); + public ReadOnlySpan FirstSpan + { + get + { + if (_startObject is null) + return default; + var memory = FirstMemory; + var end = ReferenceEquals(_startObject, _endObject) + ? _endIndex + : memory.Length; + return memory.Span.Slice(_startIndex, end - _startIndex); + } + } + + ReadOnlyMemory FirstMemory => _startObject switch + { + T[] array => array, + MemoryManager manager => manager.Memory, + ReadOnlySequenceSegment segment => segment.Memory, + _ => throw new UnreachableException() + }; // Returns the sequence before the index, stores the sequence after it in place. public FastReadOnlySequence SplitInPlace(long offset) { - FastReadOnlySequence prev; - - // If it's out-of-range of the first, has to resolve another segment, or is not backed by - // one array, let Slice handle it. - if (!SequenceMarshal.TryGetArray(_sequence, out var array) || array.Count <= offset) + var firstEnd = ReferenceEquals(_startObject, _endObject) + ? _endIndex + : FirstMemory.Length; + var firstLength = firstEnd - _startIndex; + if (offset == _length) { - prev = new(_sequence.Slice(0, offset), offset); - _sequence = _sequence.Slice(offset); + var exhausted = this; + this = default; + return exhausted; } - else + if (offset == firstLength + && _startObject is ReadOnlySequenceSegment segment + && segment.Next is { } next) { - Debug.Assert(offset <= int.MaxValue); - var arrayInstance = array.Array!; - prev = new(new(arrayInstance, array.Offset, (int)offset), offset); - _sequence = new(arrayInstance, array.Offset + (int)offset, array.Count - (int)offset); + var boundaryPrefix = new FastReadOnlySequence( + segment, _startIndex, segment, firstEnd, offset); + _startObject = next; + _startIndex = 0; + _length -= offset; + return boundaryPrefix; } - - _length -= offset; - return prev; - } - - // TODO arrays should not go down the slow path for First and FirstSpan, the SequenceReader variant doesn't either. - // Inline to remove the write barriers. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - ReadOnlySpan GetFirstSpan(out ArraySegment array) - { - if (SequenceMarshal.TryGetArray(_sequence, out array)) + if ((ulong)offset < (uint)firstLength) { - Debug.Assert(_sequence.IsSingleSegment); - return array.AsSpan(); + var splitIndex = _startIndex + (int)offset; + var prev = new FastReadOnlySequence( + _startObject, _startIndex, _startObject, splitIndex, offset); + _startIndex = splitIndex; + _length -= offset; + return prev; } - array = default; - return _sequence.FirstSpan; + var sequence = Sequence; + var prefix = sequence.Slice(0, offset); + var remaining = sequence.Slice(offset); + var result = new FastReadOnlySequence(prefix); + this = new(remaining); + return result; } } From b81787c2608afbd5272690e754014b0914452466 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 20:48:41 +0200 Subject: [PATCH 009/136] Publish fallback sequence positions granularly --- Slon/Pg/Protocol/BackendMessageContext.cs | 45 ++++++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index b2d5201..1906a38 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -11,7 +11,7 @@ sealed class BackendMessageContext PgDecoder _decoder = null!; BackendMessageBatch _remainingBatch; BackendMessage _current; - ReadOnlySequence _currentFallbackBuffer; + FallbackBuffer _currentFallbackBuffer; short _version; const byte PriorCancellationExposure = 1 << 0; const byte BackendTermination = 1 << 1; @@ -23,6 +23,41 @@ sealed class BackendMessageContext enum PublicationState : byte { None, Current, Peeked } PublicationState _publicationState; long _currentMessageOffset; + struct FallbackBuffer + { + ReadOnlySequenceSegment? _start; + ReadOnlySequenceSegment? _end; + int _startIndex; + int _endIndex; + + public readonly bool IsEmpty => _start is null; + + public void Set(in ReadOnlySequence buffer) + { + var start = (ReadOnlySequenceSegment)buffer.Start.GetObject()!; + var end = (ReadOnlySequenceSegment)buffer.End.GetObject()!; + if (!ReferenceEquals(_start, start)) + _start = start; + if (!ReferenceEquals(_end, end)) + _end = end; + _startIndex = buffer.Start.GetInteger() & int.MaxValue; + _endIndex = buffer.End.GetInteger() & int.MaxValue; + } + + public void Clear() + { + if (_start is not null) + _start = null; + if (_end is not null) + _end = null; + _startIndex = 0; + _endIndex = 0; + } + + public readonly ReadOnlySequence Sequence + => new(_start!, _startIndex, _end!, _endIndex); + } + public BackendMessage Current { @@ -64,15 +99,15 @@ internal void SetCurrentFallbackBuffer( in ReadOnlySequence buffer, bool required) { if (required) - _currentFallbackBuffer = buffer; + _currentFallbackBuffer.Set(in buffer); else if (!_currentFallbackBuffer.IsEmpty) - _currentFallbackBuffer = default; + _currentFallbackBuffer.Clear(); } internal ReadOnlySequence GetFallbackBuffer(short token) { Validate(token); - return _currentFallbackBuffer; + return _currentFallbackBuffer.Sequence; } public long GetCurrentMessageOffset(short token) @@ -262,7 +297,7 @@ public void RetireCurrentBatch() // A failed message poll preserves Current, but crossing this ownership boundary cannot. var invalidateToken = _publicationState is not PublicationState.None; _current = default; - _currentFallbackBuffer = default; + _currentFallbackBuffer.Clear(); _publicationState = PublicationState.None; _remainingBatch = default; _currentMessageOffset = 0; From 1d0038cec830aa40e680c7df70cb771eec278dc8 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 21:17:48 +0200 Subject: [PATCH 010/136] Move backend framing into the protocol read pipe --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 626 ++++-------------- .../Pg/ConcurrentHandoffInMemoryTests.cs | 4 +- Slon.Tests/Pg/RacingDisposeInMemoryTests.cs | 2 +- Slon.Tests/Pg/StoppingTokenInMemoryTests.cs | 4 +- Slon/Pg/PgClientOptions.cs | 2 +- Slon/Pg/Protocol/BackendMessageBatch.cs | 125 ++-- Slon/Pg/Protocol/BackendMessageBodyReader.cs | 13 +- Slon/Pg/Protocol/BackendMessageContext.cs | 75 ++- Slon/Pg/Protocol/CurrentMessageBuffer.cs | 10 + Slon/Pg/Protocol/PgClientProtocol.cs | 9 +- Slon/Pg/Protocol/PgDecoder.cs | 122 ++-- Slon/Pg/Protocol/ProtocolReadPipe.cs | 359 ++++++++-- Slon/Pipelines/PipeSegmentEnumerator.cs | 592 ----------------- Slon/Pipelines/StreamPipeReader.cs | 10 +- Slon/SlonDataSourceOptions.cs | 2 +- 15 files changed, 630 insertions(+), 1325 deletions(-) create mode 100644 Slon/Pg/Protocol/CurrentMessageBuffer.cs delete mode 100644 Slon/Pipelines/PipeSegmentEnumerator.cs diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index 4ee073a..dec2586 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -12,6 +12,24 @@ namespace Slon.Tests.Pg; [TestClass] public class BackendMessageStreamingTests { + sealed class SequenceSegment : ReadOnlySequenceSegment + { + public SequenceSegment(ReadOnlyMemory memory) => Memory = memory; + + public SequenceSegment Append(ReadOnlyMemory memory) + { + var next = new SequenceSegment(memory) + { + RunningIndex = RunningIndex + Memory.Length + }; + Next = next; + return next; + } + + public ReadOnlySequence To(SequenceSegment end) + => new(this, 0, end, end.Memory.Length); + } + sealed class RejectRetiredSuppliedReadReader(PipeReader inner) : PipeReader { ReadResult _activeRead; @@ -62,60 +80,6 @@ enum LifetimeAction Retire, } - // Treats the first 4 big-endian bytes as the total segment length. Fully buffered => Done (so the - // enumerator takes the deferred-consume branch, exactly the state the defect needs). - struct FixedSegmenter : IPipeSegmenter - { - public int MinimumSize => 4; - - public OperationStatus CreateSegment(in ReadOnlySequence buffer, out long segmentLength, out int segment) - { - segment = 0; - var reader = new SequenceReader(buffer); - if (!reader.TryReadBigEndian(out int len)) - { - segmentLength = 0; - return OperationStatus.NeedMoreData; - } - segmentLength = len; - if (buffer.Length < len) - return OperationStatus.NeedMoreData; - segment = len; - return OperationStatus.Done; - } - } - - struct StreamingSegmenter : IPipeSegmenter> - { - public int MinimumSize => 4; - - public OperationStatus CreateSegment(in ReadOnlySequence buffer, out long segmentLength, - out ReadOnlySequence segment) - { - var reader = new SequenceReader(buffer); - if (!reader.TryReadBigEndian(out int len)) - { - segmentLength = 0; - segment = default; - return OperationStatus.NeedMoreData; - } - - segmentLength = len; - segment = buffer.Slice(0, Math.Min(buffer.Length, len)); - return buffer.Length < len ? OperationStatus.NeedMoreData : OperationStatus.Done; - } - } - - static byte[] LenPrefixed(int total) - { - var bytes = new byte[total]; - bytes[0] = (byte)(total >> 24); - bytes[1] = (byte)(total >> 16); - bytes[2] = (byte)(total >> 8); - bytes[3] = (byte)total; - return bytes; - } - static byte[] BackendMessageBytes(BackendType type, int totalLength) { var bytes = new byte[totalLength]; @@ -131,15 +95,19 @@ static byte[] BackendMessageBytes(BackendType type, ReadOnlySpan body) return bytes; } - static PipeSegmentEnumerator BuildEnumerator(byte[] wire) + static ReadOnlySequence Segmented( + ReadOnlyMemory first, ReadOnlyMemory second) { - // A MemoryStream returns the wire bytes then 0 (EOF) on every subsequent read, so re-drives - // after completion re-hit the same terminal state the recovery drain does against a closed peer. - var reader = new DefaultStreamPipeReader( - new MemoryStream(wire, writable: false), - new StreamPipeReaderOptions(bufferSize: 8192, useZeroByteReads: false), - supportCancelPending: false); - return new(reader, new FixedSegmenter(), ownsReader: true); + var start = new SequenceSegment(first); + return start.To(start.Append(second)); + } + + static async ValueTask MoveNextBatchAsync(ProtocolReadPipe pipe) + { + pipe.PrepareMoveNextBatch(); + var read = await pipe.ReadAsync(CancellationToken.None); + return pipe.CompleteMoveNextBatch( + read, CancellationToken.None, out _); } [TestMethod] @@ -219,13 +187,15 @@ public async Task MovingToNextBatch_RetiresCurrentBeforeReturningItsStorage() { var pipe = new Pipe(); var reader = new RejectRetiredSuppliedReadReader(pipe.Reader); - var protocolPipe = new ProtocolReadPipe( - new(reader, new BackendMessageBatch.Segmenter(), ownsReader: true)); + var protocolPipe = new ProtocolReadPipe(reader, + BackendMessageBatch.DefaultDataRowStreamingThreshold, + ownsReader: true); await pipe.Writer.WriteAsync(BackendMessageBytes(BackendType.CommandComplete, 6)); - Assert.IsTrue(protocolPipe.TryMoveNextBatch(out _)); + Assert.IsTrue(await MoveNextBatchAsync(protocolPipe)); Assert.IsTrue(protocolPipe.TryMoveNext()); var accessor = protocolPipe.Current.GetAccessor(); + Assert.IsFalse(protocolPipe.TryMoveNext()); var observedAdvance = false; reader.BeforeAdvance = () => @@ -238,7 +208,7 @@ public async Task MovingToNextBatch_RetiresCurrentBeforeReturningItsStorage() }; await pipe.Writer.WriteAsync(BackendMessageBytes(BackendType.ReadyForQuery, 6)); - Assert.IsTrue(protocolPipe.TryMoveNextBatch(out _)); + Assert.IsTrue(await MoveNextBatchAsync(protocolPipe)); Assert.IsTrue(observedAdvance); Assert.IsTrue(protocolPipe.TryMoveNext()); Assert.AreEqual(BackendType.ReadyForQuery, protocolPipe.Current.Header.Type); @@ -306,113 +276,6 @@ public void BackendMessageContext_CurrentLifetime_ExhaustiveShortSequences() } } - [TestMethod] - public async Task ReDriveAfterEof_Async_ReturnsFalseWithoutCorruption() - { - var e = BuildEnumerator(LenPrefixed(24)); - - Assert.IsTrue(await e.MoveNextAsync(), "first segment should be produced"); - Assert.AreEqual(24, e.Current); - Assert.IsFalse(await e.MoveNextAsync(), "second call consumes the segment and reaches EOF"); - - // The recovery drain keeps pulling after completion; before the fix each of these re-applied the - // stale deferred advance and threw ArgumentOutOfRangeException('length') from the ReadResult build. - for (var i = 0; i < 6; i++) - Assert.IsFalse(await e.MoveNextAsync(), $"re-drive #{i} past completion must stay false"); - - await e.DisposeAsync(); - } - - [TestMethod] - public async Task CompletedRead_WithFinalBufferedSegment_StillPublishesIt() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator(pipe.Reader, new FixedSegmenter()); - await pipe.Writer.WriteAsync(LenPrefixed(24)); - await pipe.Writer.CompleteAsync(); - - Assert.IsTrue(await e.MoveNextAsync()); - Assert.AreEqual(24, e.Current); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task CompletedRead_WithTruncatedFinalSegment_Throws() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator(pipe.Reader, new FixedSegmenter()); - await pipe.Writer.WriteAsync(LenPrefixed(24).AsMemory(0, 12)); - await pipe.Writer.CompleteAsync(); - - Assert.IsTrue(await e.MoveNextAsync(), "the useful prefix is published before its missing tail is discovered"); - await Assert.ThrowsExactlyAsync(async () => await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task CompletedRead_AfterPendingContinuation_Throws() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator>( - pipe.Reader, new StreamingSegmenter()); - await pipe.Writer.WriteAsync(LenPrefixed(24).AsMemory(0, 12)); - - Assert.IsTrue(await e.MoveNextAsync()); - Assert.IsFalse(e.TryContinueCurrentSegment(e.Current.End, e.Current.Length, out _)); - await pipe.Writer.CompleteAsync(); - - await Assert.ThrowsExactlyAsync(async () => await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task CompletionAtEveryByteInsideSegment_Throws() - { - const int segmentLength = 32; - var wire = LenPrefixed(segmentLength); - - for (var cut = 1; cut < segmentLength; cut++) - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator(pipe.Reader, new FixedSegmenter()); - await pipe.Writer.WriteAsync(wire.AsMemory(0, cut)); - await pipe.Writer.CompleteAsync(); - - await Assert.ThrowsExactlyAsync(async () => - { - while (await e.MoveNextAsync()) { } - }, $"completion at byte {cut} must not become clean EOF"); - await e.DisposeAsync(); - } - } - - [TestMethod] - public async Task CompletionAtEveryByteInsideSuccessor_ThrowsAfterPredecessor() - { - const int firstLength = 12; - const int secondLength = 32; - var wire = new byte[firstLength + secondLength]; - LenPrefixed(firstLength).CopyTo(wire, 0); - LenPrefixed(secondLength).CopyTo(wire, firstLength); - - for (var successorBytes = 1; successorBytes < secondLength; successorBytes++) - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator(pipe.Reader, new FixedSegmenter()); - await pipe.Writer.WriteAsync(wire.AsMemory(0, firstLength + successorBytes)); - await pipe.Writer.CompleteAsync(); - - Assert.IsTrue(await e.MoveNextAsync(), $"predecessor missing at successor byte {successorBytes}"); - Assert.AreEqual(firstLength, e.Current); - await Assert.ThrowsExactlyAsync(async () => - { - while (await e.MoveNextAsync()) { } - }, $"completion at successor byte {successorBytes} must not become clean EOF"); - await e.DisposeAsync(); - } - } - [TestMethod] public async Task RepeatedQueryFrames_WithSmallRecycledBuffers_NeverEnterMessageBodies() { @@ -434,22 +297,23 @@ public async Task RepeatedQueryFrames_WithSmallRecycledBuffers_NeverEnterMessage new MemoryStream(wire, writable: false), new StreamPipeReaderOptions(bufferSize: 1024, useZeroByteReads: false), supportCancelPending: false); - var e = new PipeSegmentEnumerator( - reader, new BackendMessageBatch.Segmenter(), ownsReader: true); + var readPipe = new ProtocolReadPipe(reader, + BackendMessageBatch.DefaultDataRowStreamingThreshold, + ownsReader: true); var messageIndex = 0; - while (await e.MoveNextAsync()) + while (await readPipe.MoveNextAsync(default)) { - var batch = e.Current; - while (batch.TryReadNextInPlace(out var header, out _, out _)) + while (readPipe.TryMoveNext()) { - Assert.AreEqual(response[messageIndex % response.Length][0], (byte)header.Type, + Assert.AreEqual(response[messageIndex % response.Length][0], + (byte)readPipe.Current.Header.Type, $"message {messageIndex}"); messageIndex++; } } Assert.AreEqual(repetitions * response.Length, messageIndex); - await e.DisposeAsync(); + await readPipe.DisposeAsync(); } [TestMethod] @@ -473,44 +337,45 @@ public async Task RepeatedQueryFrames_ThroughDirectReads_NeverEnterMessageBodies new MemoryStream(wire, writable: false), new StreamPipeReaderOptions(bufferSize: 1024, useZeroByteReads: false), supportCancelPending: false); - var e = new PipeSegmentEnumerator( - reader, new BackendMessageBatch.Segmenter(), ownsReader: true); + var readPipe = new ProtocolReadPipe(reader, + BackendMessageBatch.DefaultDataRowStreamingThreshold, + ownsReader: true); + var directReader = (StreamPipeReader)readPipe.PipeReader; var messageIndex = 0; while (true) { - if (e.TryMoveNext(out var completed)) - { - ValidateBatch(e.Current); - continue; - } - if (completed) - break; - - Assert.IsTrue(e.TryBeginDirectRead(default, out var read)); + readPipe.PrepareMoveNextBatch(); + Assert.IsTrue(directReader.SupportsDirectRead); + var read = directReader.BeginDirectRead(default); while (true) { var length = await read; - if (e.CompleteDirectRead(length, default, out read, out var readFinished, out completed)) + if (!directReader.CompleteDirectRead(length, default, out read, out var result)) + { + continue; + } + if (readPipe.CompleteMoveNextBatch( + result, default, out var completed)) { - ValidateBatch(e.Current); + ValidateBatch(); break; } - if (!readFinished) - continue; - Assert.IsTrue(completed); - goto done; + if (completed) + goto done; + break; } } done: Assert.AreEqual(repetitions * response.Length, messageIndex); - await e.DisposeAsync(); + await readPipe.DisposeAsync(); - void ValidateBatch(BackendMessageBatch batch) + void ValidateBatch() { - while (batch.TryReadNextInPlace(out var header, out _, out _)) + while (readPipe.TryMoveNext()) { - Assert.AreEqual(response[messageIndex % response.Length][0], (byte)header.Type, + Assert.AreEqual(response[messageIndex % response.Length][0], + (byte)readPipe.Current.Header.Type, $"message {messageIndex}"); messageIndex++; } @@ -543,214 +408,21 @@ static byte[][] QueryResponseBytes() public async Task Eof_InvalidatesPublishedBackendMessage() { var pipe = new Pipe(); - var batches = new PipeSegmentEnumerator( - pipe.Reader, new BackendMessageBatch.Segmenter()); - var readPipe = new ProtocolReadPipe(batches); + var readPipe = new ProtocolReadPipe(pipe.Reader, + BackendMessageBatch.DefaultDataRowStreamingThreshold); await pipe.Writer.WriteAsync(BackendMessageBytes(BackendType.ReadyForQuery, 6)); Assert.IsTrue(await readPipe.MoveNextAsync(CancellationToken.None)); - readPipe.CommitBatch(); Assert.IsTrue(readPipe.TryMoveNext()); var accessor = readPipe.Current.GetAccessor(); + Assert.IsFalse(readPipe.TryMoveNext()); await pipe.Writer.CompleteAsync(); Assert.IsFalse(await readPipe.MoveNextAsync(CancellationToken.None)); Assert.IsFalse(readPipe.TryGetCurrent(out _)); Assert.ThrowsExactly(() => _ = readPipe.Current); Assert.ThrowsExactly(() => _ = accessor.Message); - await batches.DisposeAsync(); - } - - [TestMethod] - public void ReDriveAfterEof_Sync_ReturnsFalseWithoutCorruption() - { - var e = BuildEnumerator(LenPrefixed(24)); - - Assert.IsTrue(e.MoveNext(), "first segment should be produced"); - Assert.AreEqual(24, e.Current); - Assert.IsFalse(e.MoveNext(), "second call consumes the segment and reaches EOF"); - - for (var i = 0; i < 6; i++) - Assert.IsFalse(e.MoveNext(), $"re-drive #{i} past completion must stay false"); - - e.Dispose(); - } - - [TestMethod] - public async Task TryMoveNext_PollsFragmentedSegmentWithoutSuspending() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator(pipe.Reader, new FixedSegmenter()); - - Assert.IsFalse(e.TryMoveNext(out var completed)); - Assert.IsFalse(completed); - - var wire = LenPrefixed(24); - await pipe.Writer.WriteAsync(wire.AsMemory(0, 2)); - Assert.IsFalse(e.TryMoveNext(out completed), "a partial header must request another read"); - Assert.IsFalse(completed); - - await pipe.Writer.WriteAsync(wire.AsMemory(2)); - Assert.IsTrue(e.TryMoveNext(out completed)); - Assert.IsFalse(completed); - Assert.AreEqual(24, e.Current); - - await pipe.Writer.CompleteAsync(); - Assert.IsFalse(e.TryMoveNext(out completed)); - Assert.IsTrue(completed); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task DirectRead_PreservesFramingAndTerminalState() - { - var e = BuildEnumerator(LenPrefixed(24)); - - Assert.IsTrue(e.TryBeginDirectRead(default, out var read)); - var length = await read; - Assert.IsTrue(e.CompleteDirectRead(length, default, out _, out var readFinished, out var completed)); - Assert.IsTrue(readFinished); - Assert.IsFalse(completed); - Assert.AreEqual(24, e.Current); - - Assert.IsFalse(e.TryMoveNext(out completed)); - Assert.IsFalse(completed); - Assert.IsTrue(e.TryBeginDirectRead(default, out read)); - length = await read; - Assert.IsFalse(e.CompleteDirectRead(length, default, out _, out readFinished, out completed)); - Assert.IsTrue(readFinished); - Assert.IsTrue(completed); - - await e.DisposeAsync(); - } - - [TestMethod] - public async Task ContinueCurrentSegment_StreamsWithoutCrossingNextSegment() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator>( - pipe.Reader, new StreamingSegmenter()); - var first = LenPrefixed(12); - var second = LenPrefixed(8); - - await pipe.Writer.WriteAsync(first.AsMemory(0, 6)); - Assert.IsTrue(e.TryMoveNext(out _)); - Assert.AreEqual(6, e.Current.Length); - - Assert.IsFalse(e.TryContinueCurrentSegment(e.Current.End, e.Current.Length, out _)); - - var tail = new byte[first.Length - 6 + second.Length]; - first.AsSpan(6).CopyTo(tail); - second.CopyTo(tail.AsSpan(first.Length - 6)); - await pipe.Writer.WriteAsync(tail); - - Assert.IsTrue(e.TryContinueCurrentSegment(e.Current.End, e.Current.Length, out var continuation)); - Assert.IsTrue(continuation.IsComplete); - Assert.AreEqual(6, continuation.Buffer.Length); - - Assert.IsTrue(e.TryMoveNext(out _)); - Assert.AreEqual(8, e.Current.Length); - - await pipe.Writer.CompleteAsync(); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task MoveNextAsync_SkipsUnconsumedPartialSegmentBeforeReadingNext() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator>( - pipe.Reader, new StreamingSegmenter()); - var first = LenPrefixed(12); - var second = LenPrefixed(8); - - await pipe.Writer.WriteAsync(first.AsMemory(0, 6)); - Assert.IsTrue(await e.MoveNextAsync()); - Assert.AreEqual(6, e.Current.Length); - - byte[] remaining = [.. first.AsSpan(6), .. second]; - await pipe.Writer.WriteAsync(remaining); - Assert.IsTrue(await e.MoveNextAsync()); - Assert.AreEqual(second.Length, e.Current.Length); - - await pipe.Writer.CompleteAsync(); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task TryMoveNext_SkipsUnconsumedPartialSegmentBeforePollingNext() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator>( - pipe.Reader, new StreamingSegmenter()); - var first = LenPrefixed(12); - var second = LenPrefixed(8); - - await pipe.Writer.WriteAsync(first.AsMemory(0, 6)); - Assert.IsTrue(e.TryMoveNext(out _)); - Assert.AreEqual(6, e.Current.Length); - - byte[] remaining = [.. first.AsSpan(6), .. second]; - await pipe.Writer.WriteAsync(remaining); - Assert.IsTrue(e.TryMoveNext(out _)); - Assert.AreEqual(second.Length, e.Current.Length); - - await pipe.Writer.CompleteAsync(); - Assert.IsFalse(e.TryMoveNext(out var completed)); - Assert.IsTrue(completed); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task ContinueCurrentSegmentAsync_PreservesPartialProgress() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator>( - pipe.Reader, new StreamingSegmenter()); - var wire = LenPrefixed(12); - - await pipe.Writer.WriteAsync(wire.AsMemory(0, 6)); - Assert.IsTrue(e.TryMoveNext(out _)); - - var pending = e.ContinueCurrentSegmentAsync(e.Current.End, e.Current.Length); - Assert.IsFalse(pending.IsCompleted); - await pipe.Writer.WriteAsync(wire.AsMemory(6, 3)); - var middle = await pending; - Assert.IsFalse(middle.IsComplete); - Assert.AreEqual(3, middle.Buffer.Length); - - pending = e.ContinueCurrentSegmentAsync(middle.Buffer.End, middle.Buffer.Length); - await pipe.Writer.WriteAsync(wire.AsMemory(9)); - var final = await pending; - Assert.IsTrue(final.IsComplete); - Assert.AreEqual(3, final.Buffer.Length); - - await pipe.Writer.CompleteAsync(); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task ExtendCurrentSegmentAsync_RetainsUntilTheCompleteSegment() - { - var wire = LenPrefixed(128 * 1024); - var reader = new DefaultStreamPipeReader( - new MemoryStream(wire, writable: false), - new StreamPipeReaderOptions(bufferSize: 8192, useZeroByteReads: false), - supportCancelPending: false); - var e = new PipeSegmentEnumerator>( - reader, new StreamingSegmenter(), ownsReader: true); - - Assert.IsTrue(await e.MoveNextAsync()); - CurrentSegmentBuffer current; - do current = await e.ExtendCurrentSegmentAsync(); - while (!current.IsComplete); - - Assert.AreEqual(wire.Length, current.Buffer.Length); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); + await readPipe.DisposeAsync(); } [TestMethod] @@ -761,13 +433,12 @@ public async Task BackendBodyReader_ExtendsPrefixThenSlides() for (var i = BackendHeader.ByteCount; i < wire.Length; i++) wire[i] = (byte)i; - var segments = new PipeSegmentEnumerator( - pipe.Reader, new BackendMessageBatch.Segmenter(8)); - var decoder = new PgDecoder(segments, CancellationToken.None, Timeout.InfiniteTimeSpan); + var decoder = new PgDecoder( + pipe.Reader, 8, CancellationToken.None, Timeout.InfiniteTimeSpan); decoder.Pipe.BindDecoder(decoder); await pipe.Writer.WriteAsync(wire.AsMemory(0, 8)); - Assert.IsTrue(decoder.Pipe.TryMoveNextBatch(out _)); + Assert.IsTrue(await MoveNextBatchAsync(decoder.Pipe)); Assert.IsTrue(decoder.Pipe.TryMoveNext()); var body = decoder.Pipe.Current.OpenBodyReader(); Assert.AreEqual(3, body.Buffer.Length); @@ -789,7 +460,7 @@ public async Task BackendBodyReader_ExtendsPrefixThenSlides() } [TestMethod] - public async Task BackendSegmenter_ExtendedRowAdvancesToTrailingMessage() + public async Task BackendReadPipe_ExtendedRowAdvancesToTrailingMessage() { var bind = BackendMessageBytes(BackendType.BindComplete, BackendHeader.ByteCount); var row = BackendMessageBytes(BackendType.DataRow, 128 * 1024); @@ -798,102 +469,33 @@ public async Task BackendSegmenter_ExtendedRowAdvancesToTrailingMessage() bind.CopyTo(wire, 0); row.CopyTo(wire, bind.Length); complete.CopyTo(wire, bind.Length + row.Length); - var reader = new DefaultStreamPipeReader( - new MemoryStream(wire, writable: false), - new StreamPipeReaderOptions(bufferSize: 64 * 1024, useZeroByteReads: false), - supportCancelPending: false); - var e = new PipeSegmentEnumerator( - reader, new BackendMessageBatch.Segmenter(), ownsReader: true); - - Assert.IsTrue(await e.MoveNextAsync()); - CurrentSegmentBuffer current; - do current = await e.ExtendCurrentSegmentAsync(); - while (!current.IsComplete); - - Assert.IsTrue(await e.MoveNextAsync()); - Assert.IsTrue(e.Current.TryReadNextInPlace(out var header, out _, out _)); - Assert.AreEqual(BackendType.CommandComplete, header.Type); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task ContinueCurrentSegment_SlidesPastOnlyTheConsumedPrefix() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator>( - pipe.Reader, new StreamingSegmenter()); - var wire = LenPrefixed(12); - for (var i = 4; i < wire.Length; i++) - wire[i] = (byte)i; - - await pipe.Writer.WriteAsync(wire.AsMemory(0, 8)); - Assert.IsTrue(e.TryMoveNext(out _)); - var consumed = e.Current.GetPosition(6); - Assert.IsFalse(e.TryContinueCurrentSegment(consumed, 6, out _)); - - await pipe.Writer.WriteAsync(wire.AsMemory(8, 2)); - Assert.IsTrue(e.TryContinueCurrentSegment(consumed, 6, out var middle)); - Assert.IsFalse(middle.IsComplete); - CollectionAssert.AreEqual(wire.AsSpan(6, 4).ToArray(), middle.Buffer.ToArray()); - - await pipe.Writer.WriteAsync(wire.AsMemory(10)); - var final = await e.ContinueCurrentSegmentAsync(middle.Buffer.End, middle.Buffer.Length); - Assert.IsTrue(final.IsComplete); - CollectionAssert.AreEqual(wire.AsSpan(10).ToArray(), final.Buffer.ToArray()); - - await pipe.Writer.CompleteAsync(); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task MoveNext_CanDrainAfterAContinuationPollParks() - { - var pipe = new Pipe(); - var e = new PipeSegmentEnumerator>( - pipe.Reader, new StreamingSegmenter()); - var first = LenPrefixed(12); - var second = LenPrefixed(8); - - await pipe.Writer.WriteAsync(first.AsMemory(0, 6)); - Assert.IsTrue(e.TryMoveNext(out _)); - Assert.IsFalse(e.TryContinueCurrentSegment(e.Current.End, e.Current.Length, out _)); - - await pipe.Writer.WriteAsync(first.AsMemory(6)); - await pipe.Writer.WriteAsync(second); - Assert.IsTrue(await e.MoveNextAsync(), "normal iteration must take over the pending continuation read"); - Assert.AreEqual(second.Length, e.Current.Length); - - await pipe.Writer.CompleteAsync(); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); - } - - [TestMethod] - public async Task SuppliedRead_IsInspectedBeforeItsConsumedPrefixIsRetired() - { - var pipe = new Pipe(); - var reader = new RejectRetiredSuppliedReadReader(pipe.Reader); - var e = new PipeSegmentEnumerator>( - reader, new StreamingSegmenter()); - var first = LenPrefixed(12); - var second = LenPrefixed(8); - - await pipe.Writer.WriteAsync(first.AsMemory(0, 6)); - Assert.IsTrue(e.TryMoveNext(out _)); - Assert.IsFalse(e.TryMoveNext(out _), "the poll should advance once and park for the tail"); - - await pipe.Writer.WriteAsync(first.AsMemory(6)); - await pipe.Writer.WriteAsync(second); - var supplied = await e.ReadAsync(CancellationToken.None); - reader.RejectAdvanceAtActiveStart(); + var pipe = new Pipe(new PipeOptions( + pauseWriterThreshold: 256 * 1024, + resumeWriterThreshold: 128 * 1024)); + var decoder = new PgDecoder(pipe.Reader, + BackendMessageBatch.DefaultDataRowStreamingThreshold, + CancellationToken.None, Timeout.InfiniteTimeSpan); + decoder.Pipe.BindDecoder(decoder); - Assert.IsTrue(e.TryMoveNext(supplied, CancellationToken.None, out _)); - Assert.AreEqual(second.Length, e.Current.Length); + var initialLength = bind.Length + + BackendMessageBatch.DefaultDataRowStreamingThreshold; + await pipe.Writer.WriteAsync(wire.AsMemory(0, initialLength)); + Assert.IsTrue(await decoder.Pipe.MoveNextAsync(default)); + Assert.IsTrue(decoder.Pipe.TryMoveNext()); + Assert.AreEqual(BackendType.BindComplete, decoder.Pipe.Current.Header.Type); + Assert.IsTrue(decoder.Pipe.TryMoveNext()); + Assert.AreEqual(BackendType.DataRow, decoder.Pipe.Current.Header.Type); + var body = decoder.Pipe.Current.OpenBodyReader(); + await pipe.Writer.WriteAsync(wire.AsMemory(initialLength)); + while (!body.IsComplete) + Assert.IsTrue(body.TryExtend()); + Assert.IsFalse(decoder.Pipe.TryMoveNext()); + Assert.IsTrue(await decoder.Pipe.MoveNextAsync(default)); + Assert.IsTrue(decoder.Pipe.TryMoveNext()); + Assert.AreEqual(BackendType.CommandComplete, decoder.Pipe.Current.Header.Type); await pipe.Writer.CompleteAsync(); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); + await ((IAsyncDisposable)decoder).DisposeAsync(); } [TestMethod] @@ -907,50 +509,44 @@ public void BackendMessage_BufferedRequiresTagAndDeclaredLength() } [TestMethod] - public void BackendSegmenter_WaitsForUsefulPartialDataRowPrefix() + public void BackendBatch_WaitsForUsefulPartialDataRowPrefix() { var rowLength = 128 * 1024; var wire = BackendMessageBytes(BackendType.DataRow, rowLength); - var segmenter = new BackendMessageBatch.Segmenter(); var smallPrefix = new ReadOnlySequence(wire.AsMemory(0, 32)); - Assert.AreEqual(OperationStatus.NeedMoreData, - segmenter.CreateSegment(smallPrefix, out var length, out _)); - Assert.AreEqual(0, length); - Assert.AreEqual(BackendMessageBatch.Segmenter.DefaultDataRowStreamingThreshold, segmenter.MinimumSize); + var batch = new BackendMessageBatch(smallPrefix); + Assert.IsFalse(batch.TryReadNextInPlace(out _, out _, out _)); + Assert.AreEqual(BackendMessageBatch.DefaultDataRowStreamingThreshold, + batch.RequiredBufferedLength); var usefulPrefix = new ReadOnlySequence( - wire.AsMemory(0, BackendMessageBatch.Segmenter.DefaultDataRowStreamingThreshold)); - Assert.AreEqual(OperationStatus.Done, - segmenter.CreateSegment(usefulPrefix, out length, out var batch)); - Assert.AreEqual(rowLength, length); + wire.AsMemory(0, BackendMessageBatch.DefaultDataRowStreamingThreshold)); + batch = new(usefulPrefix); Assert.IsTrue(batch.TryReadNextInPlace(out var rowHeader, out var partialRow, out _)); Assert.AreEqual(BackendType.DataRow, rowHeader.Type); - Assert.AreEqual(BackendMessageBatch.Segmenter.DefaultDataRowStreamingThreshold, partialRow.Length); + Assert.AreEqual(BackendMessageBatch.DefaultDataRowStreamingThreshold, partialRow.Length); Assert.IsFalse(new BackendMessage(rowHeader, partialRow, new BackendMessageContext(), 0).Buffered); } [TestMethod] - public void BackendSegmenter_FramesUnknownMessageType() + public void BackendBatch_FramesUnknownMessageType() { var wire = BackendHeaderBytes((BackendType)(byte)'o', 4); - var segmenter = new BackendMessageBatch.Segmenter(); + var batch = new BackendMessageBatch(new ReadOnlySequence(wire)); - Assert.AreEqual(OperationStatus.Done, - segmenter.CreateSegment(new ReadOnlySequence(wire), out var length, out var batch)); - Assert.AreEqual(wire.Length, length); Assert.IsTrue(batch.TryReadNextInPlace(out var header, out _, out _)); Assert.AreEqual((BackendType)(byte)'o', header.Type); } [TestMethod] - public void BackendSegmenter_RejectsMessageBeyondPostgreSqlAllocationLimit() + public void BackendBatch_RejectsMessageBeyondPostgreSqlAllocationLimit() { var wire = BackendHeaderBytes(BackendType.DataRow, 0x3FFF_FFFF); - var segmenter = new BackendMessageBatch.Segmenter(); + var batch = new BackendMessageBatch(new ReadOnlySequence(wire)); Assert.ThrowsExactly(() => - segmenter.CreateSegment(new ReadOnlySequence(wire), out _, out _)); + batch.TryReadNextInPlace(out _, out _, out _)); } static byte[] BackendHeaderBytes(BackendType type, int length) diff --git a/Slon.Tests/Pg/ConcurrentHandoffInMemoryTests.cs b/Slon.Tests/Pg/ConcurrentHandoffInMemoryTests.cs index 36a61e3..5f3c763 100644 --- a/Slon.Tests/Pg/ConcurrentHandoffInMemoryTests.cs +++ b/Slon.Tests/Pg/ConcurrentHandoffInMemoryTests.cs @@ -250,8 +250,8 @@ public override void WaitUntilWritable(TimeSpan timeout) { } public EchoServerTransport(byte[] handshake, byte[] response) { _response = response; - // Slon's PipeSegmentEnumerator requires its OWN StreamPipeReader for the sync path - // (not the BCL PipeReader.Create). Wrap the in-memory pipe as a stream, same as the socket transport. + // Slon's synchronous protocol path requires a StreamPipeReader (not the BCL + // PipeReader.Create). Wrap the in-memory pipe as a stream, like the socket transport. _clientReader = new Slon.Pipelines.DefaultStreamPipeReader( _toClient.Reader.AsStream(), new StreamPipeReaderOptions(bufferSize: 8192, useZeroByteReads: false), diff --git a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs index 9b3f911..2a2c939 100644 --- a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs +++ b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs @@ -143,7 +143,7 @@ public async Task GracefulCloseMidRowEofSurfacesClosedException() for (var i = 0; i < dataRowIndex; i++) transport.ReleaseSegment(messages[i]); transport.ReleaseSegment(messages[dataRowIndex] - .AsSpan(0, BackendMessageBatch.Segmenter.DefaultDataRowStreamingThreshold).ToArray()); + .AsSpan(0, BackendMessageBatch.DefaultDataRowStreamingThreshold).ToArray()); Assert.IsTrue(await resultPending); var rows = flowEnumerator.Current.GetAsyncEnumerator(CommandResult.RowBuffering.Streaming); diff --git a/Slon.Tests/Pg/StoppingTokenInMemoryTests.cs b/Slon.Tests/Pg/StoppingTokenInMemoryTests.cs index 2da723d..56ee86c 100644 --- a/Slon.Tests/Pg/StoppingTokenInMemoryTests.cs +++ b/Slon.Tests/Pg/StoppingTokenInMemoryTests.cs @@ -437,8 +437,8 @@ public Task ArmReadPark() return tcs.Task; } - // The protocol's transport read (PipeSegmentEnumerator) funnels through ReadAsync, including - // via the base ReadAtLeastAsync default; signaling here covers both. AdvanceTo is delegated + // The protocol transport funnels through ReadAsync, including via the base + // ReadAtLeastAsync default; signaling here covers both. AdvanceTo is delegated // verbatim (no teeing), so the base default's position tracking stays correct. public override ValueTask ReadAsync(CancellationToken cancellationToken = default) { diff --git a/Slon/Pg/PgClientOptions.cs b/Slon/Pg/PgClientOptions.cs index c73cf1a..5ed135e 100644 --- a/Slon/Pg/PgClientOptions.cs +++ b/Slon/Pg/PgClientOptions.cs @@ -43,7 +43,7 @@ public PostgreSqlSslOptions Ssl public TimeSpan ConnectionTimeout { get; init; } = Timeout.InfiniteTimeSpan; internal PgSessionResetOptions SessionReset { get; init; } = new(); - internal int DataRowStreamingThreshold { get; init; } = BackendMessageBatch.Segmenter.DefaultDataRowStreamingThreshold; + internal int DataRowStreamingThreshold { get; init; } = BackendMessageBatch.DefaultDataRowStreamingThreshold; internal int MaxInFlightFlowsPerWire { get; init; } internal PipelineScheduler? ExecutionScheduler { get; init; } diff --git a/Slon/Pg/Protocol/BackendMessageBatch.cs b/Slon/Pg/Protocol/BackendMessageBatch.cs index c1be909..3b374d9 100644 --- a/Slon/Pg/Protocol/BackendMessageBatch.cs +++ b/Slon/Pg/Protocol/BackendMessageBatch.cs @@ -9,25 +9,56 @@ namespace Slon.Pg.Protocol; // Note: both the batch and the segmenter are perf sensitive. struct BackendMessageBatch(ReadOnlySequence buffer) { + public const int DefaultDataRowStreamingThreshold = 16 * 1024; + const uint MaxMessageLength = 0x3FFF_FFFF; + FastReadOnlySequence _buffer = new(buffer); long _initialLength = buffer.Length; + readonly int _dataRowStreamingThreshold = DefaultDataRowStreamingThreshold; + long _requiredBufferedLength; + + internal BackendMessageBatch( + ReadOnlySequence buffer, int dataRowStreamingThreshold) : this(buffer) + => _dataRowStreamingThreshold = dataRowStreamingThreshold; + + BackendMessageBatch(ReadOnlySequence buffer, + int dataRowStreamingThreshold, long initialLength) + : this(buffer, dataRowStreamingThreshold) + => _initialLength = initialLength; + + public readonly long ConsumedLength => _initialLength - _buffer.Length; + public readonly long RequiredBufferedLength => _requiredBufferedLength; + public readonly SequencePosition UnreadStart => _buffer.Sequence.Start; public readonly long GetCurrentMessageOffset(long currentBufferedLength) => _initialLength - _buffer.Length - currentBufferedLength; public readonly BackendMessageBatch Slice(long offset) { - var result = new BackendMessageBatch(_buffer.Sequence.Slice(offset)); - result._initialLength = _initialLength; - return result; + return new(_buffer.Sequence.Slice(offset), + _dataRowStreamingThreshold, _initialLength); } public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) { if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) && !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader)) { - // We use default(ROSeq) - which is fully supported - as ROSeq.Empty weirdly enough wraps an empty array. - _buffer = default; + _requiredBufferedLength = ConsumedLength + Header.ByteCount; + buffer = default; + bufferLength = default; + header = default; + return false; + } + + var backendType = (BackendType)protoHeader.Tag; + if (protoHeader.MessageLength > MaxMessageLength) + throw new PgFramingException($"PostgreSQL backend message length {protoHeader.MessageLength} exceeds the maximum supported length."); + var required = backendType is BackendType.DataRow + ? Math.Min(protoHeader.MessageLength, (uint)_dataRowStreamingThreshold) + : protoHeader.MessageLength; + if (_buffer.Length < required) + { + _requiredBufferedLength = ConsumedLength + required; buffer = default; bufferLength = default; header = default; @@ -35,6 +66,7 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence - { - public const int DefaultDataRowStreamingThreshold = 16 * 1024; - const uint MaxMessageLength = 0x3FFF_FFFF; - - readonly int _dataRowStreamingThreshold; - int _minimumSize; - public int MinimumSize => _minimumSize; - - public Segmenter() : this(DefaultDataRowStreamingThreshold) {} - - public Segmenter(int dataRowStreamingThreshold) - { - ArgumentOutOfRangeException.ThrowIfNegative(dataRowStreamingThreshold); - _dataRowStreamingThreshold = dataRowStreamingThreshold; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public OperationStatus CreateSegment(in ReadOnlySequence buffer, out long segmentLength, out BackendMessageBatch segment) - { - _minimumSize = Header.ByteCount; - var reader = new SequenceReader(buffer); - var messages = 0; - var needMoreData = false; - segmentLength = 0; - - // Try span first before accessing the sequence. - while (Header.TryParse(reader.UnreadSpan, out var header) || Header.TryParseMultiSegment(reader.UnreadSequence, out header)) - { - var backendType = (BackendType)header.Tag; - if (header.MessageLength > MaxMessageLength) - throw new PgFramingException($"PostgreSQL backend message length {header.MessageLength} exceeds the maximum supported length."); - - if (reader.Remaining < header.MessageLength) - { - var required = RequiredBufferedLength(backendType, header.MessageLength); - if (reader.Remaining < required) - { - // MinimumSize is relative to the entire unconsumed pipe buffer, including messages - // already framed before this one. - _minimumSize = int.CreateSaturating(segmentLength + required); - needMoreData = true; - break; - } - - reader.Advance(reader.Remaining); - } - else - { - reader.Advance(header.MessageLength); - } - - messages++; - segmentLength += header.MessageLength; - } - - if (messages is 0) - { - segment = default; - return OperationStatus.NeedMoreData; - } - - segment = new(reader.Length == segmentLength ? buffer : buffer.Slice(0, reader.Position)); - return needMoreData ? OperationStatus.NeedMoreData : OperationStatus.Done; - } - - uint RequiredBufferedLength(BackendType backendType, uint messageLength) => backendType switch - { - BackendType.DataRow => Math.Min(messageLength, (uint)_dataRowStreamingThreshold), - // BackendType.RowDescription or - // BackendType.CopyData or - // BackendType.FunctionCallResponse or - // BackendType.NotificationResponse or - // BackendType.ParameterDescription => false, - _ => messageLength, - }; - } - // Keeps the public position components scalar so consuming the first segment does not repeatedly // reconstruct and rediscover the backing of a ReadOnlySequence. Materialize one only at API seams. struct FastReadOnlySequence @@ -215,7 +168,9 @@ public FastReadOnlySequence SplitInPlace(long offset) if (offset == _length) { var exhausted = this; - this = default; + _startObject = _endObject; + _startIndex = _endIndex; + _length = 0; return exhausted; } if (offset == firstLength diff --git a/Slon/Pg/Protocol/BackendMessageBodyReader.cs b/Slon/Pg/Protocol/BackendMessageBodyReader.cs index d293293..1ecbc15 100644 --- a/Slon/Pg/Protocol/BackendMessageBodyReader.cs +++ b/Slon/Pg/Protocol/BackendMessageBodyReader.cs @@ -56,7 +56,7 @@ public void AdvanceTo(SequencePosition consumed, long consumedLength) public bool TryRead() { EnsureAdvanced(); - if (!_context.TryContinue(_token, _consumed, _consumedLength, out var result)) + if (!_context.TrySlide(_token, _consumed, _consumedLength, out var result)) return false; Publish(result); return true; @@ -65,7 +65,8 @@ public bool TryRead() public ValueTask ReadAsync(CancellationToken cancellationToken = default) { EnsureAdvanced(); - var task = _context.ContinueAsync(_token, _consumed, _consumedLength, cancellationToken); + var task = _context.SlideAsync( + _token, _consumed, _consumedLength, cancellationToken); if (task.IsCompletedSuccessfully) { Publish(task.Result); @@ -73,14 +74,14 @@ public ValueTask ReadAsync(CancellationToken cancellationToken = default) } return Core(task); - async ValueTask Core(ValueTask task) + async ValueTask Core(ValueTask task) => Publish(await task.ConfigureAwait(false)); } public void Read() { EnsureAdvanced(); - Publish(_context.Continue(_token, _consumed, _consumedLength)); + Publish(_context.Slide(_token, _consumed, _consumedLength)); } public bool TryExtend() @@ -103,7 +104,7 @@ public ValueTask ExtendAsync(CancellationToken cancellationToken = default) } return Core(task); - async ValueTask Core(ValueTask task) + async ValueTask Core(ValueTask task) => Publish(await task.ConfigureAwait(false), retained: true); } @@ -191,7 +192,7 @@ void EnsureAdvanced() ThrowHelper.ThrowInvalidOperation("AdvanceTo must be called before reading more message data."); } - void Publish(CurrentSegmentBuffer result, bool retained = false) + void Publish(CurrentMessageBuffer result, bool retained = false) { _buffer = result.Buffer; IsComplete = result.IsComplete; diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 1906a38..a63f7a3 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -10,6 +10,7 @@ sealed class BackendMessageContext { PgDecoder _decoder = null!; BackendMessageBatch _remainingBatch; + bool _hasBatch; BackendMessage _current; FallbackBuffer _currentFallbackBuffer; short _version; @@ -128,27 +129,27 @@ public void BindDecoder(PgDecoder decoder) _decoder = decoder; } - public bool TryContinue(short token, SequencePosition consumed, long consumedLength, - out CurrentSegmentBuffer result) + public bool TrySlide(short token, SequencePosition consumed, long consumedLength, + out CurrentMessageBuffer result) { MarkBodyWindowAdvanced(token); - return _decoder.TryContinueCurrentMessage(consumed, consumedLength, out result); + return _decoder.TrySlideCurrentMessage(consumed, consumedLength, out result); } - public ValueTask ContinueAsync(short token, SequencePosition consumed, + public ValueTask SlideAsync(short token, SequencePosition consumed, long consumedLength, CancellationToken cancellationToken) { MarkBodyWindowAdvanced(token); - return _decoder.ContinueCurrentMessageAsync(consumed, consumedLength, cancellationToken); + return _decoder.SlideCurrentMessageAsync(consumed, consumedLength, cancellationToken); } - public CurrentSegmentBuffer Continue(short token, SequencePosition consumed, long consumedLength) + public CurrentMessageBuffer Slide(short token, SequencePosition consumed, long consumedLength) { MarkBodyWindowAdvanced(token); - return _decoder.ContinueCurrentMessage(consumed, consumedLength); + return _decoder.SlideCurrentMessage(consumed, consumedLength); } - public bool TryExtend(short token, out CurrentSegmentBuffer result) + public bool TryExtend(short token, out CurrentMessageBuffer result) { EnsureBodyWindowAvailable(token); if (!_decoder.TryExtendCurrentMessage(out result)) @@ -157,19 +158,19 @@ public bool TryExtend(short token, out CurrentSegmentBuffer result) return true; } - public async ValueTask ExtendAsync(short token, CancellationToken cancellationToken) + public async ValueTask ExtendAsync(short token, CancellationToken cancellationToken) { EnsureBodyWindowAvailable(token); return GetBodyBuffer(token, await _decoder.ExtendCurrentMessageAsync(cancellationToken).ConfigureAwait(false)); } - public CurrentSegmentBuffer Extend(short token) + public CurrentMessageBuffer Extend(short token) { EnsureBodyWindowAvailable(token); return GetBodyBuffer(token, _decoder.ExtendCurrentMessage()); } - CurrentSegmentBuffer GetBodyBuffer(short token, CurrentSegmentBuffer result) + CurrentMessageBuffer GetBodyBuffer(short token, CurrentMessageBuffer result) { Validate(token); var bodyOffset = _currentMessageOffset + BackendHeader.ByteCount; @@ -177,7 +178,15 @@ CurrentSegmentBuffer GetBodyBuffer(short token, CurrentSegmentBuffer result) var bufferedLength = Math.Min(bodyLength, result.Buffer.Length - bodyOffset); var body = result.Buffer.Slice(bodyOffset, bufferedLength); if (result.IsComplete) - SetCurrentFromSegment(token, result.Buffer); + { + _decoder.CompleteCurrentMessage(); + var messageLength = _current.Header.MessageLength; + var message = result.Buffer.Slice(_currentMessageOffset, messageLength); + BackendMessage.Initialize( + ref _current, _current.Header, message, this, token, buffered: true); + var messageEnd = _currentMessageOffset + messageLength; + _remainingBatch = new BackendMessageBatch(result.Buffer).Slice(messageEnd); + } return new(body, result.IsComplete); } @@ -196,16 +205,10 @@ public void EnsureBodyWindowAvailable(short token) _ = GetCurrentMessageOffset(token); } - public void SetBuffered(short token, ReadOnlySequence buffer) - { - Validate(token); - BackendMessage.Initialize(ref _current, _current.Header, buffer, this, token, buffered: true); - } - public void BufferCurrentMessage(short token) { EnsureBodyWindowAvailable(token); - CurrentSegmentBuffer result; + CurrentMessageBuffer result; do result = Extend(token); while (!result.IsComplete); } @@ -217,18 +220,12 @@ public ValueTask BufferCurrentMessageAsync(short token, CancellationToken cancel async ValueTask Core(short token, CancellationToken cancellationToken) { - CurrentSegmentBuffer result; + CurrentMessageBuffer result; do result = await ExtendAsync(token, cancellationToken).ConfigureAwait(false); while (!result.IsComplete); } } - void SetCurrentFromSegment(short token, ReadOnlySequence segment) - { - var message = segment.Slice(_currentMessageOffset, _current.Header.MessageLength); - SetBuffered(token, message); - } - void Validate(short token) { if (_publicationState is PublicationState.Peeked || _version != token) @@ -280,6 +277,10 @@ public bool TryMoveNext() if (!_remainingBatch.TryReadNextInPlace(out var header, out var buffer, out var bufferLength)) return false; ResetMessageState(); + if (bufferLength < header.MessageLength) + _decoder.SetCurrentMessageLength( + _remainingBatch.ConsumedLength - bufferLength + + header.MessageLength); BackendMessage.Initialize(ref _current, header, buffer, this, ++_version, bufferLength >= header.MessageLength); _publicationState = PublicationState.Current; @@ -300,12 +301,29 @@ public void RetireCurrentBatch() _currentFallbackBuffer.Clear(); _publicationState = PublicationState.None; _remainingBatch = default; + _hasBatch = false; _currentMessageOffset = 0; _messageState = 0; if (invalidateToken) _version++; } + public bool TryGetBatchReadRequirement( + out SequencePosition consumed, out long requiredLength) + { + if (!_hasBatch || _remainingBatch.RequiredBufferedLength <= 0) + { + consumed = default; + requiredLength = 0; + return false; + } + + consumed = _remainingBatch.UnreadStart; + requiredLength = _remainingBatch.RequiredBufferedLength + - _remainingBatch.ConsumedLength; + return true; + } + // Reads the next message WITHOUT publishing it as Current. The remaining batch cursor // really advances past the header, but the parsed (header, buffer) lands in the peek // slot and the follow-up TryMoveNext picks it up without re-parsing. The returned @@ -323,6 +341,10 @@ public bool TryPeekNext(out BackendHeader header) { return false; } + if (bufferLength < header.MessageLength) + _decoder.SetCurrentMessageLength( + _remainingBatch.ConsumedLength - bufferLength + + header.MessageLength); _messageState = 0; BackendMessage.Initialize(ref _current, header, buffer, this, ++_version, bufferLength >= header.MessageLength); @@ -346,6 +368,7 @@ public void SetBatch(BackendMessageBatch batch) "The prior batch must be retired before publishing replacement storage."); _publicationState = PublicationState.None; _remainingBatch = batch; + _hasBatch = true; } } diff --git a/Slon/Pg/Protocol/CurrentMessageBuffer.cs b/Slon/Pg/Protocol/CurrentMessageBuffer.cs new file mode 100644 index 0000000..8ed975e --- /dev/null +++ b/Slon/Pg/Protocol/CurrentMessageBuffer.cs @@ -0,0 +1,10 @@ +using System.Buffers; + +namespace Slon.Pg.Protocol; + +readonly struct CurrentMessageBuffer( + ReadOnlySequence buffer, bool isComplete) +{ + public ReadOnlySequence Buffer { get; } = buffer; + public bool IsComplete { get; } = isComplete; +} diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index 55f326e..8bdd046 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -86,7 +86,7 @@ public PgClientProtocolOptions(PgClientOptions options) public TimeSpan CancellationTimeout { get; set; } = TimeSpan.FromSeconds(10); public TimeSpan CancellationRetryInterval { get; set; } = TimeSpan.FromSeconds(1); internal PgSessionResetOptions SessionReset { get; set; } = new(); - public int DataRowStreamingThreshold { get; set; } = BackendMessageBatch.Segmenter.DefaultDataRowStreamingThreshold; + public int DataRowStreamingThreshold { get; set; } = BackendMessageBatch.DefaultDataRowStreamingThreshold; public int MaxInFlightFlowsPerWire { get; set; } public ILoggerFactory LoggerFactory { get; set; } = NullLoggerFactory.Instance; // Datasource bootstrap supplies this. A standalone raw protocol can omit backend identity and @@ -177,7 +177,6 @@ await _protocol.StartAsync(_transport, _flow, hosting, cancellationToken) TransportConnection _connection = null!; IOutputWriter _pipeWriter = null!; ProtocolDataWriter _protocolDataWriter = null!; - PipeSegmentEnumerator _pipeSegmentEnumerator = null!; PgDecoder _pgDecoder = null!; LoadObserver? _loadObserver; @@ -307,9 +306,9 @@ void Initialize(TransportConnection connection, Hosting hosting) _pipeWriter = connection.Writer as IOutputWriter ?? new PipeOutputWriter(connection.Writer); _protocolDataWriter = new(_pipeWriter, PgClientOptions.PreStartupEncoding, connection.WaitUntilWritable, AbortToken, FlowControl, _options.WriteTimeout); - _pipeSegmentEnumerator = new(connection.Reader, - new(_options.DataRowStreamingThreshold), ownsReader: true); - _pgDecoder = new(_pipeSegmentEnumerator, AbortToken, _options.ReadTimeout, _options.ReadTimeoutArmed); + _pgDecoder = new(connection.Reader, _options.DataRowStreamingThreshold, + AbortToken, _options.ReadTimeout, _options.ReadTimeoutArmed, + ownsReader: true); _admissionAvailable = hosting.AdmissionAvailable; _loadObserver = hosting.LoadObserver; diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 4b46abb..65269f7 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -19,6 +19,7 @@ namespace Slon.Pg.Protocol; public sealed class PgDecoder: IEnumerator, IAsyncEnumerator { readonly ProtocolReadPipe _pipe; + readonly StreamPipeReader? _directReader; readonly CancellationToken _abortToken; readonly TimeSpan _defaultReadTimeout; readonly Action? _readTimeoutArmed; @@ -86,6 +87,7 @@ TimeSpan GetRemainingTimeout() Action? readTimeoutArmed) { _pipe = pipe; + _directReader = pipe.PipeReader as StreamPipeReader; _abortToken = abortToken; _defaultReadTimeout = defaultReadTimeout; _readTimeout = defaultReadTimeout; @@ -95,15 +97,59 @@ TimeSpan GetRemainingTimeout() SetRemainingTimeout(Timeout.InfiniteTimeSpan); } - internal PgDecoder(PipeSegmentEnumerator messageBatchEnumerator, - CancellationToken abortToken, TimeSpan defaultReadTimeout, Action? readTimeoutArmed = null) - : this(new ProtocolReadPipe(messageBatchEnumerator), abortToken, defaultReadTimeout, readTimeoutArmed) + internal PgDecoder(PipeReader reader, int dataRowStreamingThreshold, + CancellationToken abortToken, TimeSpan defaultReadTimeout, + Action? readTimeoutArmed = null, bool ownsReader = false) + : this(new ProtocolReadPipe( + reader, dataRowStreamingThreshold, ownsReader), + abortToken, defaultReadTimeout, readTimeoutArmed) { } internal ProtocolReadPipe Pipe => _pipe; internal Encoding ClientEncoding => _control.ClientEncoding; + internal void SetCurrentMessageLength(long messageLength) + => _pipe.SetCurrentMessageLength(messageLength); + + internal void CompleteCurrentMessage() + => _pipe.CompleteCurrentMessage(); + + void PrepareMoveNextBatch() => _pipe.PrepareMoveNextBatch(); + + bool CompleteMoveNextBatch( + in ReadResult result, CancellationToken cancellationToken, out bool completed) + => _pipe.CompleteMoveNextBatch( + result, cancellationToken, out completed); + + bool MoveNextBatch(TimeSpan timeout) + => _pipe.MoveNext(timeout); + + bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask task) + { + if (_directReader is { SupportsDirectRead: true } directReader) + { + task = directReader.BeginDirectRead(cancellationToken); + return true; + } + task = default; + return false; + } + + bool CompleteDirectRead(int length, CancellationToken cancellationToken, + out ValueTask next, out bool readFinished, out bool completed) + { + if (!_directReader!.CompleteDirectRead(length, cancellationToken, out next, out var result)) + { + readFinished = false; + completed = false; + return false; + } + readFinished = true; + return CompleteMoveNextBatch(result, cancellationToken, out completed); + } + + void AbortDirectRead() => _directReader!.AbortDirectRead(); // Builds a scope-bound shell over the shared pipe with the scope's abort token. internal static PgDecoder CreateScopeShell(PgDecoder baseShell, CancellationToken abortToken, TimeSpan defaultReadTimeout) => new(baseShell._pipe, abortToken, defaultReadTimeout, baseShell._readTimeoutArmed); @@ -143,19 +189,19 @@ public void UseReadTimeout(TimeSpan timeout) void RestoreDefaultReadTimeout() => _readTimeout = _defaultReadTimeout; - internal bool TryContinueCurrentMessage(SequencePosition consumed, long consumedLength, out CurrentSegmentBuffer result) - => _pipe.TryContinueCurrentMessage(consumed, consumedLength, out result); + internal bool TrySlideCurrentMessage(SequencePosition consumed, long consumedLength, out CurrentMessageBuffer result) + => _pipe.TrySlideCurrentMessage(consumed, consumedLength, out result); - internal ValueTask ContinueCurrentMessageAsync( + internal ValueTask SlideCurrentMessageAsync( SequencePosition consumed, long consumedLength, CancellationToken cancellationToken) { EnsureUsableCts(); - if (_pipe.TryContinueCurrentMessage(consumed, consumedLength, out var result)) + if (_pipe.TrySlideCurrentMessage(consumed, consumedLength, out var result)) return new(result); return Core(cancellationToken); - async ValueTask Core(CancellationToken cancellationToken) + async ValueTask Core(CancellationToken cancellationToken) { var timeoutSet = false; var frontierFlow = EnterCancellationReadFrontier(); @@ -165,7 +211,7 @@ async ValueTask Core(CancellationToken cancellationToken) { ArmReadTimeout(); timeoutSet = true; - return await _pipe.ContinueCurrentMessageAsync( + return await _pipe.SlideCurrentMessageAsync( consumed, consumedLength, _cancellationTokenSource.Token).ConfigureAwait(false); } catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) @@ -186,7 +232,7 @@ async ValueTask Core(CancellationToken cancellationToken) } } - internal CurrentSegmentBuffer ContinueCurrentMessage( + internal CurrentMessageBuffer SlideCurrentMessage( SequencePosition consumed, long consumedLength) { var timeoutSet = false; @@ -194,7 +240,8 @@ internal CurrentSegmentBuffer ContinueCurrentMessage( { ArmReadTimeout(); timeoutSet = true; - return _pipe.ContinueCurrentMessage(consumed, consumedLength, GetRemainingTimeout()); + return _pipe.SlideCurrentMessage( + consumed, consumedLength, GetRemainingTimeout()); } catch (Exception) when (_abortToken.IsCancellationRequested && _control.ClosedException is not null) { @@ -211,10 +258,10 @@ internal CurrentSegmentBuffer ContinueCurrentMessage( } } - internal bool TryExtendCurrentMessage(out CurrentSegmentBuffer result) + internal bool TryExtendCurrentMessage(out CurrentMessageBuffer result) => _pipe.TryExtendCurrentMessage(out result); - internal ValueTask ExtendCurrentMessageAsync(CancellationToken cancellationToken) + internal ValueTask ExtendCurrentMessageAsync(CancellationToken cancellationToken) { EnsureUsableCts(); if (_pipe.TryExtendCurrentMessage(out var result)) @@ -222,7 +269,7 @@ internal ValueTask ExtendCurrentMessageAsync(CancellationT return Core(cancellationToken); - async ValueTask Core(CancellationToken cancellationToken) + async ValueTask Core(CancellationToken cancellationToken) { var timeoutSet = false; var frontierFlow = EnterCancellationReadFrontier(); @@ -253,7 +300,7 @@ async ValueTask Core(CancellationToken cancellationToken) } } - internal CurrentSegmentBuffer ExtendCurrentMessage() + internal CurrentMessageBuffer ExtendCurrentMessage() { var timeoutSet = false; try @@ -394,17 +441,14 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau return new(true); } - if (pipe.TryMoveNextBatch(out var completed)) - continue; - if (completed) - return new(ReadCompleted()); + PrepareMoveNextBatch(); var readToken = _cancellationTokenSource.Token; var frontierFlow = EnterCancellationReadFrontier(); try { retryRead: - if (pipe.TryBeginDirectRead(readToken, out var directReadTask)) + if (TryBeginDirectRead(readToken, out var directReadTask)) { try { @@ -412,7 +456,9 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau { if (!directReadTask.IsCompletedSuccessfully) return MoveNextAsyncCore(null, directReadTask, null, cancellationToken, frontierFlow); - if (pipe.CompleteDirectRead(directReadTask.Result, readToken, out directReadTask, out var readFinished, out var directReadCompleted)) + if (CompleteDirectRead(directReadTask.Result, readToken, + out directReadTask, out var readFinished, + out var directReadCompleted)) break; if (!readFinished) continue; @@ -428,7 +474,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau } catch { - pipe.AbortDirectRead(); + AbortDirectRead(); throw; } } @@ -437,7 +483,9 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau if (!readTask.IsCompletedSuccessfully) return MoveNextAsyncCore(readTask, null, null, cancellationToken, frontierFlow); LeaveCancellationReadFrontier(frontierFlow); - if (pipe.TryMoveNextBatch(readTask.Result, _cancellationTokenSource.Token, out var readCompleted)) + if (CompleteMoveNextBatch( + readTask.Result, _cancellationTokenSource.Token, + out var readCompleted)) continue; if (readCompleted) return new(ReadCompleted()); @@ -489,7 +537,9 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa LeaveCancellationReadFrontier(frontierFlow!); frontierFlow = null; readTask = null; - if (_pipe.TryMoveNextBatch(result, _cancellationTokenSource.Token, out var readCompleted)) + if (CompleteMoveNextBatch( + result, _cancellationTokenSource.Token, + out var readCompleted)) continue; if (readCompleted) return ReadCompleted(); @@ -514,7 +564,10 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa timeoutSet = true; } var length = await pendingDirectRead.ConfigureAwait(false); - if (_pipe.CompleteDirectRead(length, _cancellationTokenSource.Token, out var nextDirectRead, out var readFinished, out var readCompleted)) + if (CompleteDirectRead(length, + _cancellationTokenSource.Token, + out var nextDirectRead, out var readFinished, + out var readCompleted)) { LeaveCancellationReadFrontier(frontierFlow!); frontierFlow = null; @@ -534,7 +587,7 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa } catch (Exception ex) { - _pipe.AbortDirectRead(); + AbortDirectRead(); if (frontierFlow is not null) { LeaveCancellationReadFrontier(frontierFlow); @@ -562,16 +615,13 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa if (messageHandledTask.HasValue) continue; - if (_pipe.TryMoveNextBatch(out var completed)) - continue; - if (completed) - return ReadCompleted(); + PrepareMoveNextBatch(); try { var token = _cancellationTokenSource.Token; frontierFlow = EnterCancellationReadFrontier(); - if (_pipe.TryBeginDirectRead(token, out var nextDirectRead)) + if (TryBeginDirectRead(token, out var nextDirectRead)) directReadTask = nextDirectRead; else readTask = _pipe.ReadAsync(token); @@ -717,11 +767,8 @@ or PgTypes.BackendType.NotificationResponse return true; } - // The current batch is exhausted. Descend through any bytes the PipeReader already owns - // before reporting unavailable; only a genuinely pending physical read should make the - // async caller install its continuation tree. - if (!_pipe.TryMoveNextBatch(out _)) - break; + PrepareMoveNextBatch(); + break; } unavailable: @@ -804,12 +851,11 @@ bool MoveNextSlow() { LeaveCancellationReadFrontier(frontierFlow); } - pipe.CommitBatch(); if (!success) return ReadCompleted(); if (!TryMoveNext(pipe)) - ThrowHelper.ThrowInvalidOperation("No message in a new batch"); + continue; } // HandleMessageAuto is always sync-completing (every branch returns a diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index 140dea1..e29b0f8 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -1,106 +1,369 @@ +using System.Buffers; using System.IO.Pipelines; +using System.Runtime.CompilerServices; using Slon.Pipelines; namespace Slon.Pg.Protocol; -// Shared per-protocol read-side wire state. One instance per protocol, behind any number of -// PgDecoder shells (the base protocol shell plus a per-exclusive-scope shell). The single-pump -// invariant means only one shell ever drives this pipe at a time, so the batch enumerator and -// message context are safe to share. Token-bearing concerns (CTS, abort translation, read-timeout -// countdown, CurrentExecutionControl, and framing/handler loops) live in the shell; the pipe -// exposes only the raw read primitives and message iteration. -sealed class ProtocolReadPipe(PipeSegmentEnumerator messageBatchEnumerator) +// Shared per-protocol read-side wire state. One cursor parses each backend message once, and an +// incomplete message directly drives the next PipeReader grant. +sealed class ProtocolReadPipe( + PipeReader reader, int dataRowStreamingThreshold, bool ownsReader = false) { + enum PendingRead : byte { None, Batch, Slide, Extend } + readonly BackendMessageContext _messageContext = new(); + ReadOnlySequence _activeBuffer; + SequencePosition _examined; + SequencePosition _retainedStart; + long _currentMessageOffset; + long _currentMessageLength = -1; + long _pendingBatchOffset; + long _pendingSkipLength; + int _minimumReadSize; + PendingRead _pendingRead; + bool _hasActiveRead; + public PipeReader PipeReader => reader; public BackendMessage Current => _messageContext.Current; public bool CurrentIsError => _messageContext.CurrentIsError; - public bool TryGetCurrent(out BackendMessage message) => _messageContext.TryGetCurrent(out message); + public bool TryGetCurrent(out BackendMessage message) + => _messageContext.TryGetCurrent(out message); public bool TryMoveNext() => _messageContext.TryMoveNext(); - public bool TryPeekNext(out BackendHeader header) => _messageContext.TryPeekNext(out header); + public bool TryPeekNext(out BackendHeader header) + => _messageContext.TryPeekNext(out header); public BackendMessage Peeked => _messageContext.Peeked; public void BindDecoder(PgDecoder decoder) => _messageContext.BindDecoder(decoder); - public bool TryMoveNextBatch(out bool completed) + public void PrepareMoveNextBatch() { + if (_pendingRead is PendingRead.Batch) + return; + if (_pendingRead is not PendingRead.None) + ThrowHelper.ThrowInvalidOperation( + "The current message still has a pending read."); + + if (!_hasActiveRead) + { + _pendingBatchOffset = 0; + _minimumReadSize = BackendHeader.ByteCount; + _pendingRead = PendingRead.Batch; + return; + } + + if (_currentMessageLength > 0) + { + PrepareAfterPartialMessage(); + return; + } + + if (!_messageContext.TryGetBatchReadRequirement( + out var unread, out var requiredLength)) + ThrowHelper.ThrowInvalidOperation( + "The current backend-message batch has not been exhausted."); + + _pendingBatchOffset = 0; _messageContext.RetireCurrentBatch(); - if (!messageBatchEnumerator.TryMoveNext(out completed)) - return false; - CommitBatch(); - return true; + reader.AdvanceTo(unread, _examined); + _hasActiveRead = false; + _activeBuffer = default; + _currentMessageLength = -1; + _currentMessageOffset = 0; + _minimumReadSize = int.CreateSaturating(requiredLength); + _pendingRead = PendingRead.Batch; + } + + void PrepareAfterPartialMessage() + { + var current = _activeBuffer.Slice(_currentMessageOffset); + _messageContext.RetireCurrentBatch(); + if (current.Length >= _currentMessageLength) + { + var unread = current.GetPosition(_currentMessageLength); + reader.AdvanceTo(unread, unread); + _pendingBatchOffset = 0; + _minimumReadSize = BackendHeader.ByteCount; + } + else + { + _pendingSkipLength = _currentMessageLength - current.Length; + reader.AdvanceTo(_activeBuffer.End, _examined); + _pendingBatchOffset = _pendingSkipLength; + _minimumReadSize = int.CreateSaturating( + _pendingSkipLength + BackendHeader.ByteCount); + } + + _hasActiveRead = false; + _activeBuffer = default; + _currentMessageLength = -1; + _currentMessageOffset = 0; + _pendingRead = PendingRead.Batch; } public ValueTask ReadAsync(CancellationToken cancellationToken) - => messageBatchEnumerator.ReadAsync(cancellationToken); + => _minimumReadSize > 0 + ? reader.ReadAtLeastAsync(_minimumReadSize, cancellationToken) + : reader.ReadAsync(cancellationToken); + + public bool CompleteMoveNextBatch( + in ReadResult result, CancellationToken cancellationToken, + out bool completed) + { + if (_pendingRead is not PendingRead.Batch) + ThrowHelper.ThrowInvalidOperation("No batch read is pending."); + _pendingRead = PendingRead.None; + _minimumReadSize = 0; + if (result.IsCanceled) + ThrowHelper.ThrowOperationCanceled(cancellationToken); + if (result.Buffer.IsEmpty && result.IsCompleted) + { + completed = true; + _messageContext.RetireCurrentBatch(); + return false; + } + if (result.Buffer.IsEmpty) + { + completed = false; + return false; + } - public bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask task) - => messageBatchEnumerator.TryBeginDirectRead(cancellationToken, out task); + _activeBuffer = result.Buffer; + _examined = result.Buffer.End; + _retainedStart = result.Buffer.Start; + _hasActiveRead = true; + if (_pendingSkipLength > 0) + { + if (result.Buffer.Length < _pendingSkipLength) + { + if (result.IsCompleted) + throw new EndOfStreamException( + "The pipe completed within a backend message."); + _pendingSkipLength -= result.Buffer.Length; + reader.AdvanceTo(result.Buffer.End, result.Buffer.End); + _hasActiveRead = false; + _activeBuffer = default; + _pendingBatchOffset = _pendingSkipLength; + _minimumReadSize = int.CreateSaturating( + _pendingSkipLength + BackendHeader.ByteCount); + _pendingRead = PendingRead.Batch; + completed = false; + return false; + } + _pendingBatchOffset = _pendingSkipLength; + _pendingSkipLength = 0; + } + _currentMessageOffset = _pendingBatchOffset; + var batchBuffer = _pendingBatchOffset is 0 + ? result.Buffer + : result.Buffer.Slice(_pendingBatchOffset); + if (batchBuffer.IsEmpty) + { + completed = result.IsCompleted; + if (completed) + _messageContext.RetireCurrentBatch(); + if (!completed) + { + reader.AdvanceTo(result.Buffer.End, result.Buffer.End); + _hasActiveRead = false; + _activeBuffer = default; + _pendingBatchOffset = 0; + _minimumReadSize = BackendHeader.ByteCount; + _pendingRead = PendingRead.Batch; + } + return false; + } + _currentMessageLength = -1; + var batch = new BackendMessageBatch( + batchBuffer, dataRowStreamingThreshold); + _messageContext.SetBatch(batch); + completed = false; + return true; + } - public bool CompleteDirectRead(int length, CancellationToken cancellationToken, out ValueTask next, out bool readFinished, out bool completed) + public bool TrySlideCurrentMessage( + SequencePosition consumed, long consumedLength, + out CurrentMessageBuffer result) { - if (!messageBatchEnumerator.CompleteDirectRead(length, cancellationToken, out next, out readFinished, out completed)) + PrepareCurrentMessageRead(consumed, consumedLength, PendingRead.Slide); + if (!reader.TryRead(out var read)) + { + result = default; return false; - CommitBatch(); + } + result = CompleteCurrentMessageRead(read); return true; } - public void AbortDirectRead() => messageBatchEnumerator.AbortDirectRead(); + public ValueTask SlideCurrentMessageAsync( + SequencePosition consumed, long consumedLength, + CancellationToken cancellationToken) + { + PrepareCurrentMessageRead(consumed, consumedLength, PendingRead.Slide); + return CompleteCurrentMessageReadAsync( + reader.ReadAsync(cancellationToken), cancellationToken); + } - public bool TryMoveNextBatch(ReadResult result, CancellationToken cancellationToken, out bool completed) + public CurrentMessageBuffer SlideCurrentMessage( + SequencePosition consumed, long consumedLength, TimeSpan timeout) { - _messageContext.RetireCurrentBatch(); - if (!messageBatchEnumerator.TryMoveNext(result, cancellationToken, out completed)) + if (reader is not StreamPipeReader syncReader) + throw new NotSupportedException( + "Underlying pipe reader does not support synchronous reads."); + PrepareCurrentMessageRead(consumed, consumedLength, PendingRead.Slide); + return CompleteCurrentMessageRead(syncReader.Read(timeout)); + } + public bool TryExtendCurrentMessage(out CurrentMessageBuffer result) + { + PrepareCurrentMessageRead( + _retainedStart, consumedLength: 0, PendingRead.Extend); + if (!reader.TryRead(out var read)) + { + result = default; return false; - CommitBatch(); + } + result = CompleteCurrentMessageRead(read); return true; } - public bool TryContinueCurrentMessage(SequencePosition consumed, long consumedLength, out CurrentSegmentBuffer result) - => messageBatchEnumerator.TryContinueCurrentSegment(consumed, consumedLength, out result); + public ValueTask ExtendCurrentMessageAsync( + CancellationToken cancellationToken) + { + PrepareCurrentMessageRead( + _retainedStart, consumedLength: 0, PendingRead.Extend); + return CompleteCurrentMessageReadAsync( + reader.ReadAsync(cancellationToken), cancellationToken); + } - public ValueTask ContinueCurrentMessageAsync( - SequencePosition consumed, long consumedLength, CancellationToken cancellationToken) - => messageBatchEnumerator.ContinueCurrentSegmentAsync(consumed, consumedLength, cancellationToken); + public CurrentMessageBuffer ExtendCurrentMessage(TimeSpan timeout) + { + if (reader is not StreamPipeReader syncReader) + throw new NotSupportedException( + "Underlying pipe reader does not support synchronous reads."); + PrepareCurrentMessageRead( + _retainedStart, consumedLength: 0, PendingRead.Extend); + return CompleteCurrentMessageRead(syncReader.Read(timeout)); + } - public CurrentSegmentBuffer ContinueCurrentMessage( - SequencePosition consumed, long consumedLength, TimeSpan timeout) - => messageBatchEnumerator.ContinueCurrentSegment(consumed, consumedLength, timeout); + void PrepareCurrentMessageRead( + SequencePosition consumed, long consumedLength, PendingRead mode) + { + if (_pendingRead == mode) + return; + if (_pendingRead is not PendingRead.None || !_hasActiveRead + || _currentMessageLength <= 0) + ThrowHelper.ThrowInvalidOperation( + "The current message is not awaiting more data."); + if (mode is PendingRead.Slide + && (consumedLength <= 0 || consumedLength >= _currentMessageLength)) + throw new ArgumentOutOfRangeException(nameof(consumedLength)); - public bool TryExtendCurrentMessage(out CurrentSegmentBuffer result) - => messageBatchEnumerator.TryExtendCurrentSegment(out result); + reader.AdvanceTo( + mode is PendingRead.Slide ? consumed : _retainedStart, + _examined); + if (mode is PendingRead.Slide) + { + _retainedStart = consumed; + _currentMessageOffset = 0; + _currentMessageLength -= consumedLength; + } + _hasActiveRead = false; + _activeBuffer = default; + _pendingRead = mode; + } - public ValueTask ExtendCurrentMessageAsync(CancellationToken cancellationToken) - => messageBatchEnumerator.ExtendCurrentSegmentAsync(cancellationToken); + ValueTask CompleteCurrentMessageReadAsync( + ValueTask task, CancellationToken cancellationToken) + => task.IsCompletedSuccessfully + ? new(CompleteCurrentMessageRead(task.Result, cancellationToken)) + : Core(task, cancellationToken); - public CurrentSegmentBuffer ExtendCurrentMessage(TimeSpan timeout) - => messageBatchEnumerator.ExtendCurrentSegment(timeout); + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask Core( + ValueTask task, CancellationToken cancellationToken) + => CompleteCurrentMessageRead( + await task.ConfigureAwait(false), cancellationToken); - public ValueTask MoveNextAsync(CancellationToken cancellationToken) + CurrentMessageBuffer CompleteCurrentMessageRead( + in ReadResult result, CancellationToken cancellationToken = default) { - _messageContext.RetireCurrentBatch(); - return messageBatchEnumerator.MoveNextAsync(cancellationToken); + if (_pendingRead is not (PendingRead.Slide or PendingRead.Extend)) + ThrowHelper.ThrowInvalidOperation( + "No current-message read is pending."); + _pendingRead = PendingRead.None; + if (result.IsCanceled) + ThrowHelper.ThrowOperationCanceled(cancellationToken); + + _activeBuffer = result.Buffer; + _examined = result.Buffer.End; + _retainedStart = result.Buffer.Start; + _hasActiveRead = true; + var current = result.Buffer.Slice(_currentMessageOffset); + if (result.IsCompleted && current.Length < _currentMessageLength) + throw new EndOfStreamException( + "The pipe completed within a backend message."); + var complete = current.Length >= _currentMessageLength; + var buffer = complete + ? current.Slice(0, _currentMessageLength) + : current; + _examined = complete ? buffer.End : result.Buffer.End; + return new(buffer, complete); + } + + public async ValueTask MoveNextAsync( + CancellationToken cancellationToken) + { + while (true) + { + PrepareMoveNextBatch(); + var read = await ReadAsync(cancellationToken).ConfigureAwait(false); + if (CompleteMoveNextBatch( + read, cancellationToken, out var completed)) + return true; + if (completed) + return false; + } } public bool MoveNext(TimeSpan timeout) { - _messageContext.RetireCurrentBatch(); - return messageBatchEnumerator.MoveNext(timeout); + if (reader is not StreamPipeReader syncReader) + throw new NotSupportedException( + "Underlying pipe reader does not support synchronous reads."); + while (true) + { + PrepareMoveNextBatch(); + var read = _minimumReadSize > 0 + ? syncReader.ReadAtLeast(_minimumReadSize, timeout) + : syncReader.Read(timeout); + if (CompleteMoveNextBatch( + read, CancellationToken.None, out var completed)) + return true; + if (completed) + return false; + } + } + + public void SetCurrentMessageLength(long messageLength) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(messageLength); + _currentMessageLength = messageLength; } - // Publishes the just-read batch as the current batch the message context iterates. - public void CommitBatch() => _messageContext.SetBatch(messageBatchEnumerator.Current); + public void CompleteCurrentMessage() + => _currentMessageLength = -1; public void Dispose() { _messageContext.RetireCurrentBatch(); - messageBatchEnumerator.Dispose(); + if (ownsReader) + reader.Complete(); } public ValueTask DisposeAsync() { _messageContext.RetireCurrentBatch(); - return messageBatchEnumerator.DisposeAsync(); + return ownsReader ? reader.CompleteAsync() : default; } } diff --git a/Slon/Pipelines/PipeSegmentEnumerator.cs b/Slon/Pipelines/PipeSegmentEnumerator.cs deleted file mode 100644 index 9f4ffbd..0000000 --- a/Slon/Pipelines/PipeSegmentEnumerator.cs +++ /dev/null @@ -1,592 +0,0 @@ -using System.Buffers; -using System.Collections; -using System.IO.Pipelines; -using System.Runtime.CompilerServices; - -namespace Slon.Pipelines; - -interface IPipeSegmenter -{ - /// - /// MinimumSize guarantees CreateSegment won't be called unless there is enough data to examine. - /// - int MinimumSize { get; } - - /// - /// Create the next segment from the given buffer, returning segment information on OperationStatus.Done. - /// - /// The buffer to try to read the next segment from. - /// The length of the segment, this may be larger than the amount buffered at the time of the call. - /// Segment to return to the caller. - /// Whether the call was successful, requires more data, or invalid data was found, DestinationTooSmall is not supported. - OperationStatus CreateSegment(in ReadOnlySequence buffer, out long segmentLength, out TSegment segment); -} - -readonly struct CurrentSegmentBuffer(ReadOnlySequence buffer, bool isComplete) -{ - public ReadOnlySequence Buffer { get; } = buffer; - public bool IsComplete { get; } = isComplete; -} - -sealed class PipeSegmentEnumerator(PipeReader reader, TSegmenter segmenter, bool ownsReader = false) - : IEnumerator, IAsyncEnumerator - where TSegmenter: IPipeSegmenter -{ - readonly StreamPipeReader? _directReader = reader as StreamPipeReader; - TSegmenter _segmenter = segmenter; - TSegment _current = default!; - - SequencePosition _examinedPosition; - SequencePosition _currentSegmentStart; - SequencePosition? _consumePosition; - long _currentLength = -1; - byte _currentSegmentReadPending; - - public PipeReader PipeReader => reader; - - public ValueTask ReadAsync(CancellationToken cancellationToken) - => reader.ReadAsync(cancellationToken); - - public bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask task) - { - if (_directReader is { SupportsDirectRead: true } directReader) - { - task = directReader.BeginDirectRead(cancellationToken); - return true; - } - task = default; - return false; - } - - public bool CompleteDirectRead(int length, CancellationToken cancellationToken, out ValueTask next, out bool readFinished, out bool completed) - { - if (!_directReader!.CompleteDirectRead(length, cancellationToken, out next, out var result)) - { - readFinished = false; - completed = false; - return false; - } - readFinished = true; - return TryMoveNext(result, cancellationToken, out completed); - } - - public void AbortDirectRead() => _directReader!.AbortDirectRead(); - - public bool TryMoveNext(ReadResult result, CancellationToken cancellationToken, out bool completed) - { - if (result.IsCanceled) - ThrowHelper.ThrowOperationCanceled(cancellationToken); - return TryMoveNext(result, hasRead: true, out completed); - } - - // The underlying reader reported completion, so the wire is at EOF. Disarm the deferred advance by - // clearing the pending-segment sentinel: a re-drive past completion (recovery drain, or any caller - // that keeps pulling after false) must not re-apply a stale consume position, whose segment and - // backing array have since been consumed and pool-recycled, driving the buffer accounting negative. - // Returns false so completion sites read as return EndOfData(). - bool EndOfData() - { - _consumePosition = null; - _examinedPosition = default; - _currentSegmentStart = default; - _currentLength = -1; - _currentSegmentReadPending = 0; - return false; - } - - static bool IsEmptyCompletion(in ReadResult result) - => result.IsCompleted && result.Buffer.IsEmpty; - - static void ThrowIfTruncatedCompletion(in ReadResult result) - { - if (result.IsCompleted) - throw new EndOfStreamException("The pipe completed within a framed segment."); - } - - ValueTask IAsyncEnumerator.MoveNextAsync() => MoveNextAsync(CancellationToken.None); - - // Nonblocking counterpart to MoveNext/MoveNextAsync. It consumes every byte already available - // from the reader and returns false only when another physical read is required. completed - // distinguishes that would-block from EOF. This is the poll primitive used by read-wake drivers: - // after one leaf wake they re-enter here and synchronously descend framing again. - public bool TryMoveNext(out bool completed) - => TryMoveNext(default, hasRead: false, out completed); - - // Releases a consumed prefix of a partially buffered segment and polls for its next bytes. The - // returned buffer never crosses the segment boundary; normal MoveNext resumes at that boundary. - public bool TryContinueCurrentSegment(SequencePosition consumed, long consumedLength, out CurrentSegmentBuffer result) - { - PrepareCurrentSegmentRead(consumed, consumedLength, mode: 1); - if (!reader.TryRead(out var readResult)) - { - result = default; - return false; - } - - result = CompleteCurrentSegmentRead(readResult); - return true; - } - - public bool TryExtendCurrentSegment(out CurrentSegmentBuffer result) - { - PrepareCurrentSegmentRead(_currentSegmentStart, consumedLength: 0, mode: 2); - if (!reader.TryRead(out var readResult)) - { - result = default; - return false; - } - - result = CompleteCurrentSegmentRead(readResult); - return true; - } - - public ValueTask ContinueCurrentSegmentAsync( - SequencePosition consumed, long consumedLength, CancellationToken cancellationToken = default) - { - PrepareCurrentSegmentRead(consumed, consumedLength, mode: 1); - var task = reader.ReadAsync(cancellationToken); - return task.IsCompletedSuccessfully - ? new(CompleteCurrentSegmentRead(task.Result, cancellationToken)) - : Core(task, cancellationToken); - - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask Core(ValueTask task, CancellationToken cancellationToken) - => CompleteCurrentSegmentRead(await task.ConfigureAwait(false), cancellationToken); - } - - public ValueTask ExtendCurrentSegmentAsync(CancellationToken cancellationToken = default) - { - PrepareCurrentSegmentRead(_currentSegmentStart, consumedLength: 0, mode: 2); - var task = reader.ReadAtLeastAsync((int)Math.Min(_currentLength, int.MaxValue), cancellationToken); - return task.IsCompletedSuccessfully - ? new(CompleteCurrentSegmentRead(task.Result, cancellationToken)) - : Core(task, cancellationToken); - - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask Core(ValueTask task, CancellationToken cancellationToken) - => CompleteCurrentSegmentRead(await task.ConfigureAwait(false), cancellationToken); - } - - public CurrentSegmentBuffer ContinueCurrentSegment( - SequencePosition consumed, long consumedLength, TimeSpan timeout = default) - { - if (reader is not StreamPipeReader syncReader) - throw new NotSupportedException("Underlying pipe reader does not support synchronous reads."); - - PrepareCurrentSegmentRead(consumed, consumedLength, mode: 1); - return CompleteCurrentSegmentRead(syncReader.Read(timeout)); - } - - public CurrentSegmentBuffer ExtendCurrentSegment(TimeSpan timeout = default) - { - if (reader is not StreamPipeReader syncReader) - throw new NotSupportedException("Underlying pipe reader does not support synchronous reads."); - - PrepareCurrentSegmentRead(_currentSegmentStart, consumedLength: 0, mode: 2); - return CompleteCurrentSegmentRead(syncReader.ReadAtLeast((int)Math.Min(_currentLength, int.MaxValue), timeout)); - } - - void PrepareCurrentSegmentRead(SequencePosition consumed, long consumedLength, byte mode) - { - if (_currentSegmentReadPending != 0) - { - if (_currentSegmentReadPending != mode) - ThrowHelper.ThrowInvalidOperation("The pending segment read uses a different continuation mode."); - return; - } - if (_currentLength <= 0 || _consumePosition is not null) - ThrowHelper.ThrowInvalidOperation("The current segment is not awaiting more data."); - if (mode == 1 && (consumedLength <= 0 || consumedLength >= _currentLength)) - throw new ArgumentOutOfRangeException(nameof(consumedLength)); - - reader.AdvanceTo(consumed, _examinedPosition); - _currentLength -= consumedLength; - _currentSegmentReadPending = mode; - } - - CurrentSegmentBuffer CompleteCurrentSegmentRead(ReadResult result, - CancellationToken cancellationToken = default) - { - _currentSegmentReadPending = 0; - if (result.IsCanceled) - ThrowHelper.ThrowOperationCanceled(cancellationToken); - if (result.IsCompleted && result.Buffer.Length < _currentLength) - throw new EndOfStreamException("The pipe completed within a framed segment."); - - var isComplete = result.Buffer.Length >= _currentLength; - var buffer = isComplete ? result.Buffer.Slice(0, _currentLength) : result.Buffer; - _currentSegmentStart = result.Buffer.Start; - _consumePosition = isComplete ? buffer.End : null; - _examinedPosition = buffer.End; - return new(buffer, isComplete); - } - - bool TryMoveNext(ReadResult suppliedRead, bool hasRead, out bool completed) - { - completed = false; - - if (_currentLength is not -1) - { - var segmentReadPending = _currentSegmentReadPending != 0; - _currentSegmentReadPending = 0; - if (_consumePosition is null) - { - // A supplied read is already the result of the advance performed by the poll which - // returned false. Advancing again would retire that result before we inspect it. - if (!segmentReadPending && !hasRead) - reader.AdvanceTo(_currentSegmentStart, _examinedPosition); - if (!TryTakeRead(out var consumeResult)) - return false; - if (IsEmptyCompletion(consumeResult)) - ThrowIfTruncatedCompletion(consumeResult); - if (consumeResult.IsCanceled) - ThrowHelper.ThrowOperationCanceled(CancellationToken.None); - if (consumeResult.Buffer.Length < _currentLength) - { - ThrowIfTruncatedCompletion(consumeResult); - reader.AdvanceTo(consumeResult.Buffer.Start, consumeResult.Buffer.End); - return false; - } - reader.AdvanceTo(consumeResult.Buffer.GetPosition(_currentLength)); - _consumePosition = null; - _currentLength = -1; - } - else - { - reader.AdvanceTo(_consumePosition.GetValueOrDefault(), _examinedPosition); - _consumePosition = null; - _currentLength = -1; - } - } - - if (!TryTakeRead(out var result)) - return false; - if (IsEmptyCompletion(result)) - { - completed = true; - return EndOfData(); - } - if (result.IsCanceled) - ThrowHelper.ThrowOperationCanceled(CancellationToken.None); - - var status = _segmenter.CreateSegment(result.Buffer, out _currentLength, out _current); - switch (status) - { - case OperationStatus.NeedMoreData when _currentLength > 0: - case OperationStatus.Done: - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_currentLength, "segmentLength"); - _consumePosition = _currentLength <= result.Buffer.Length ? result.Buffer.GetPosition(_currentLength) : null; - _currentSegmentStart = result.Buffer.Start; - _examinedPosition = _consumePosition ?? result.Buffer.End; - return true; - case OperationStatus.NeedMoreData: - ThrowIfTruncatedCompletion(result); - reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); - _currentLength = -1; - return false; - case OperationStatus.InvalidData: - reader.Complete(new Exception("Segmenter encountered invalid data.")); - completed = true; - return false; - case OperationStatus.DestinationTooSmall: - ThrowHelper.ThrowInvalidOperation(); - return default; - case var value: - ThrowHelper.ThrowUnhandledCase(value); - return default; - } - - bool TryTakeRead(out ReadResult result) - { - if (hasRead) - { - result = suppliedRead; - suppliedRead = default; - hasRead = false; - return true; - } - return reader.TryRead(out result); - } - } - - public ValueTask MoveNextAsync(CancellationToken cancellationToken = default) - { - ValueTask task; - ReadResult result; - - // Advance past current segment. - if (_currentLength is not -1) - { - var segmentReadPending = _currentSegmentReadPending != 0; - _currentSegmentReadPending = 0; - // Not everything was buffered when the segment was returned (e.g. with length prefixed segments). - if (_consumePosition is null) - { - if (!segmentReadPending) - reader.AdvanceTo(_currentSegmentStart, _examinedPosition); - task = reader.ReadAtLeastAsync((int)long.Min(_currentLength, int.MaxValue), cancellationToken); - if (!task.IsCompletedSuccessfully) - return Core(task, cancellationToken, consume: true); - result = task.Result; - if (IsEmptyCompletion(result)) - ThrowIfTruncatedCompletion(result); - if (result.IsCanceled) - return new(Task.FromException(new OperationCanceledException(cancellationToken))); - - if (result.Buffer.Length < _currentLength) - { - ThrowIfTruncatedCompletion(result); - return Core(new(result), cancellationToken, consume: true); - } - if (result.Buffer.Length > _currentLength) - return Core(new(result), cancellationToken, consume: true); - reader.AdvanceTo(result.Buffer.GetPosition(_currentLength)); - _consumePosition = null; - } - else - { - reader.AdvanceTo(_consumePosition.GetValueOrDefault(), _examinedPosition); - _consumePosition = null; - } - } - - task = reader.ReadAtLeastAsync(_segmenter.MinimumSize, cancellationToken); - if (!task.IsCompletedSuccessfully) - return Core(task, cancellationToken); - - result = task.Result; - if (IsEmptyCompletion(result)) - return new(EndOfData()); - if (result.IsCanceled) - return new(Task.FromException(new OperationCanceledException(cancellationToken))); - - var status = _segmenter.CreateSegment(result.Buffer, out _currentLength, out _current); - switch (status) - { - case OperationStatus.NeedMoreData when _currentLength > 0: - case OperationStatus.Done: - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_currentLength, "segmentLength"); - _consumePosition = _currentLength <= result.Buffer.Length ? result.Buffer.GetPosition(_currentLength) : null; - _currentSegmentStart = result.Buffer.Start; - // Stop examined at the segment boundary so trailing buffered bytes (next segment's data) stay visible to the next ReadAsync. - _examinedPosition = _consumePosition ?? result.Buffer.End; - return new(true); - case OperationStatus.DestinationTooSmall: - ThrowHelper.ThrowInvalidOperation(); - return default; - case OperationStatus.NeedMoreData: - ThrowIfTruncatedCompletion(result); - return Core(new(result), cancellationToken, needMoreData: true); - case OperationStatus.InvalidData: - return InvalidData(); - case var value: - ThrowHelper.ThrowUnhandledCase(value); - return default; - } - - - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask Core(ValueTask task, CancellationToken cancellationToken, bool consume = false, bool needMoreData = false) - { - while (true) - { - var result = await task.ConfigureAwait(false); - if (IsEmptyCompletion(result)) - { - if (consume || needMoreData) - ThrowIfTruncatedCompletion(result); - return EndOfData(); - } - if (result.IsCanceled) - ThrowHelper.ThrowOperationCanceled(cancellationToken); - - if (consume) - { - if (result.Buffer.Length < _currentLength) - { - ThrowIfTruncatedCompletion(result); - reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); - task = reader.ReadAsync(cancellationToken); - continue; - } - reader.AdvanceTo(result.Buffer.GetPosition(_currentLength)); - task = reader.ReadAtLeastAsync(_segmenter.MinimumSize, cancellationToken); - consume = false; - continue; - } - if (needMoreData) - { - reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); - _currentLength = -1; - task = reader.ReadAtLeastAsync(_segmenter.MinimumSize, cancellationToken); - needMoreData = false; - continue; - } - - var status = _segmenter.CreateSegment(result.Buffer, out _currentLength, out _current); - switch (status) - { - case OperationStatus.NeedMoreData when _currentLength > 0: - case OperationStatus.Done: - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_currentLength, "segmentLength"); - _consumePosition = _currentLength <= result.Buffer.Length ? result.Buffer.GetPosition(_currentLength) : null; - _currentSegmentStart = result.Buffer.Start; - // Stop examined at the segment boundary so trailing buffered bytes stay visible to the next ReadAsync. - _examinedPosition = _consumePosition ?? result.Buffer.End; - return true; - case OperationStatus.DestinationTooSmall: - ThrowHelper.ThrowInvalidOperation(); - return default; - case OperationStatus.NeedMoreData: - ThrowIfTruncatedCompletion(result); - needMoreData = true; - break; - case OperationStatus.InvalidData: - return await InvalidData().ConfigureAwait(false); - case var value: - ThrowHelper.ThrowUnhandledCase(value); - return default; - } - } - } - - async ValueTask InvalidData() - { - await reader.CompleteAsync(new Exception("Segmenter encountered invalid data.")).ConfigureAwait(false); - return false; - } - } - - bool IEnumerator.MoveNext() => MoveNext(default(TimeSpan)); - public bool MoveNext(TimeSpan timeout = default) - { - if (reader is not StreamPipeReader syncReader) - throw new NotSupportedException("Underlying pipe reader does not support synchronous reads."); - - ReadResult result; - var consume = false; - var needMoreData = false; - - // Advance past current segment. - if (_currentLength is not -1) - { - var segmentReadPending = _currentSegmentReadPending != 0; - _currentSegmentReadPending = 0; - if (_consumePosition is null) - { - if (!segmentReadPending) - reader.AdvanceTo(_currentSegmentStart, _examinedPosition); - result = syncReader.ReadAtLeast((int)long.Min(_currentLength, int.MaxValue), timeout); - if (IsEmptyCompletion(result)) - ThrowIfTruncatedCompletion(result); - if (result.IsCanceled) - ThrowHelper.ThrowOperationCanceled(CancellationToken.None); - - if (result.Buffer.Length < _currentLength) - { - ThrowIfTruncatedCompletion(result); - consume = true; - goto loop; - } - if (result.Buffer.Length > _currentLength) - { - consume = true; - goto loop; - } - reader.AdvanceTo(result.Buffer.GetPosition(_currentLength)); - _consumePosition = null; - } - else - { - reader.AdvanceTo(_consumePosition.GetValueOrDefault(), _examinedPosition); - _consumePosition = null; - } - } - - result = syncReader.ReadAtLeast(_segmenter.MinimumSize, timeout); - - loop: - while (true) - { - if (IsEmptyCompletion(result)) - { - if (consume || needMoreData) - ThrowIfTruncatedCompletion(result); - return EndOfData(); - } - if (result.IsCanceled) - ThrowHelper.ThrowOperationCanceled(CancellationToken.None); - - if (consume) - { - if (result.Buffer.Length < _currentLength) - { - ThrowIfTruncatedCompletion(result); - reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); - result = syncReader.Read(timeout); - continue; - } - reader.AdvanceTo(result.Buffer.GetPosition(_currentLength)); - result = syncReader.ReadAtLeast(_segmenter.MinimumSize, timeout); - consume = false; - continue; - } - if (needMoreData) - { - reader.AdvanceTo(result.Buffer.Start, result.Buffer.End); - _currentLength = -1; - result = syncReader.ReadAtLeast(_segmenter.MinimumSize, timeout); - needMoreData = false; - continue; - } - - var status = _segmenter.CreateSegment(result.Buffer, out _currentLength, out _current); - switch (status) - { - case OperationStatus.NeedMoreData when _currentLength > 0: - case OperationStatus.Done: - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(_currentLength, "segmentLength"); - _consumePosition = _currentLength <= result.Buffer.Length ? result.Buffer.GetPosition(_currentLength) : null; - _currentSegmentStart = result.Buffer.Start; - // Stop examined at the segment boundary so trailing buffered bytes stay visible to the next ReadAsync. - _examinedPosition = _consumePosition ?? result.Buffer.End; - return true; - case OperationStatus.DestinationTooSmall: - ThrowHelper.ThrowInvalidOperation(); - return default; - case OperationStatus.NeedMoreData: - ThrowIfTruncatedCompletion(result); - needMoreData = true; - break; - case OperationStatus.InvalidData: - reader.Complete(new Exception("Segmenter encountered invalid data.")); - return false; - case var value: - ThrowHelper.ThrowUnhandledCase(value); - return default; - } - } - } - - public TSegment Current - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => _current; - } - - public void Dispose() - { - if (ownsReader) - reader.Complete(); - } - - public ValueTask DisposeAsync() - { - if (ownsReader) - return reader.CompleteAsync(); - return new(); - } - - object? IEnumerator.Current => Current; - void IEnumerator.Reset() => throw new NotSupportedException(); -} diff --git a/Slon/Pipelines/StreamPipeReader.cs b/Slon/Pipelines/StreamPipeReader.cs index 2fcc725..bbe35c1 100644 --- a/Slon/Pipelines/StreamPipeReader.cs +++ b/Slon/Pipelines/StreamPipeReader.cs @@ -227,6 +227,7 @@ public override bool TryRead(out ReadResult result) // or AbortDirectRead. The protocol must interrupt and join that tenure before completing the // reader and returning its destination buffer. internal bool SupportsDirectRead => PendingReadTokenSource is null; + const int BufferedDirectRead = -1; internal void EnsureCanUpgradeStream() { @@ -247,7 +248,9 @@ internal ValueTask BeginDirectRead(CancellationToken cancellationToken) try { - if (Segments.BufferedBytes is 0 && UseZeroByteReads) + if (Segments.BufferedBytes is not 0 && !ExaminedEverything) + return new(BufferedDirectRead); + if (UseZeroByteReads) { _directReadAwaitingData = true; return Stream.ReadAsync(Memory.Empty, cancellationToken); @@ -276,12 +279,13 @@ internal bool CompleteDirectRead(int length, CancellationToken cancellationToken length = next.Result; } - if (length is not 0) + if (length > 0) { ExaminedEverything = false; Segments.Grow(length); } - result = new ReadResult(Segments.GetReadOnlySequence(), isCanceled: false, isCompleted: length is 0); + result = new ReadResult(Segments.GetReadOnlySequence(), isCanceled: false, + isCompleted: length is 0); next = default; EndStartedRead(); return true; diff --git a/Slon/SlonDataSourceOptions.cs b/Slon/SlonDataSourceOptions.cs index d1a99ad..ed4a176 100644 --- a/Slon/SlonDataSourceOptions.cs +++ b/Slon/SlonDataSourceOptions.cs @@ -112,7 +112,7 @@ public int PoolSize /// /// DataRows larger than this may cross the decoder boundary before their complete body has arrived. /// - public int DataRowStreamingThreshold { get; init; } = BackendMessageBatch.Segmenter.DefaultDataRowStreamingThreshold; + public int DataRowStreamingThreshold { get; init; } = BackendMessageBatch.DefaultDataRowStreamingThreshold; /// Configures which connection state is reset when an exclusive scope is released. internal PgSessionResetOptions SessionReset { get; init; } = new(); /// From 4adcf26cfafaa24477c42676727c630f8d8f757a Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 22:42:45 +0200 Subject: [PATCH 011/136] Rename backend message batches to cursors --- Slon.Tests/Pg/BackendMessageParsingTests.cs | 2 +- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 88 +++++++++---------- Slon.Tests/Pg/RacingDisposeInMemoryTests.cs | 2 +- Slon/Pg/PgClientOptions.cs | 2 +- Slon/Pg/Protocol/BackendMessageContext.cs | 44 +++++----- ...essageBatch.cs => BackendMessageCursor.cs} | 12 +-- Slon/Pg/Protocol/PgClientProtocol.cs | 2 +- Slon/Pg/Protocol/PgDecoder.cs | 20 ++--- Slon/Pg/Protocol/ProtocolReadPipe.cs | 76 ++++++++-------- Slon/SlonDataSourceOptions.cs | 2 +- 10 files changed, 125 insertions(+), 125 deletions(-) rename Slon/Pg/Protocol/{BackendMessageBatch.cs => BackendMessageCursor.cs} (96%) diff --git a/Slon.Tests/Pg/BackendMessageParsingTests.cs b/Slon.Tests/Pg/BackendMessageParsingTests.cs index f6f169c..cb65f8e 100644 --- a/Slon.Tests/Pg/BackendMessageParsingTests.cs +++ b/Slon.Tests/Pg/BackendMessageParsingTests.cs @@ -32,7 +32,7 @@ static BackendMessage Message(PgTypes.BackendType type, ReadOnlySpan body) BinaryPrimitives.WriteInt32BigEndian(bytes.AsSpan(1), length); body.CopyTo(bytes.AsSpan(BackendHeader.ByteCount)); var context = new BackendMessageContext(); - context.SetBatch(new BackendMessageBatch(new ReadOnlySequence(bytes))); + context.SetCursor(new BackendMessageCursor(new ReadOnlySequence(bytes))); Assert.IsTrue(context.TryMoveNext()); return context.Current; } diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index dec2586..eee0ec0 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -102,11 +102,11 @@ static ReadOnlySequence Segmented( return start.To(start.Append(second)); } - static async ValueTask MoveNextBatchAsync(ProtocolReadPipe pipe) + static async ValueTask ReadNextAsync(ProtocolReadPipe pipe) { - pipe.PrepareMoveNextBatch(); + pipe.PrepareRead(); var read = await pipe.ReadAsync(CancellationToken.None); - return pipe.CompleteMoveNextBatch( + return pipe.CompleteRead( read, CancellationToken.None, out _); } @@ -150,7 +150,7 @@ public void BackendMessageContext_CurrentThrowsOutsidePublicationWindow() Assert.IsFalse(context.TryGetCurrent(out _)); Assert.ThrowsExactly(() => _ = context.Current); - context.SetBatch(new BackendMessageBatch( + context.SetCursor(new BackendMessageCursor( new ReadOnlySequence(BackendMessageBytes(BackendType.CommandComplete, 6)))); Assert.IsTrue(context.TryMoveNext()); Assert.IsTrue(context.TryGetCurrent(out var current)); @@ -160,39 +160,39 @@ public void BackendMessageContext_CurrentThrowsOutsidePublicationWindow() Assert.IsTrue(context.TryGetCurrent(out current)); Assert.AreEqual(BackendType.CommandComplete, current.Header.Type); Assert.AreEqual(BackendType.CommandComplete, accessor.Message.Header.Type); - context.RetireCurrentBatch(); + context.RetireCursor(); Assert.IsFalse(context.TryGetCurrent(out _)); Assert.ThrowsExactly(() => _ = context.Current); Assert.ThrowsExactly(() => _ = accessor.Message); } [TestMethod] - public void BackendMessageBatch_AdvancesAcrossExactSegmentBoundary() + public void BackendMessageCursor_AdvancesAcrossExactSegmentBoundary() { var first = BackendMessageBytes(BackendType.CommandComplete, 6); var second = BackendMessageBytes(BackendType.ReadyForQuery, 6); - var batch = new BackendMessageBatch(Segmented(first, second)); + var cursor = new BackendMessageCursor(Segmented(first, second)); - Assert.IsTrue(batch.TryReadNextInPlace(out var header, out var message, out _)); + Assert.IsTrue(cursor.TryReadNextInPlace(out var header, out var message, out _)); Assert.AreEqual(BackendType.CommandComplete, header.Type); CollectionAssert.AreEqual(first, message.ToArray()); - Assert.IsTrue(batch.TryReadNextInPlace(out header, out message, out _)); + Assert.IsTrue(cursor.TryReadNextInPlace(out header, out message, out _)); Assert.AreEqual(BackendType.ReadyForQuery, header.Type); CollectionAssert.AreEqual(second, message.ToArray()); - Assert.IsFalse(batch.TryReadNextInPlace(out _, out _, out _)); + Assert.IsFalse(cursor.TryReadNextInPlace(out _, out _, out _)); } [TestMethod] - public async Task MovingToNextBatch_RetiresCurrentBeforeReturningItsStorage() + public async Task MovingToNextRead_RetiresCurrentBeforeReturningItsStorage() { var pipe = new Pipe(); var reader = new RejectRetiredSuppliedReadReader(pipe.Reader); var protocolPipe = new ProtocolReadPipe(reader, - BackendMessageBatch.DefaultDataRowStreamingThreshold, + BackendMessageCursor.DefaultDataRowStreamingThreshold, ownsReader: true); await pipe.Writer.WriteAsync(BackendMessageBytes(BackendType.CommandComplete, 6)); - Assert.IsTrue(await MoveNextBatchAsync(protocolPipe)); + Assert.IsTrue(await ReadNextAsync(protocolPipe)); Assert.IsTrue(protocolPipe.TryMoveNext()); var accessor = protocolPipe.Current.GetAccessor(); Assert.IsFalse(protocolPipe.TryMoveNext()); @@ -208,7 +208,7 @@ public async Task MovingToNextBatch_RetiresCurrentBeforeReturningItsStorage() }; await pipe.Writer.WriteAsync(BackendMessageBytes(BackendType.ReadyForQuery, 6)); - Assert.IsTrue(await MoveNextBatchAsync(protocolPipe)); + Assert.IsTrue(await ReadNextAsync(protocolPipe)); Assert.IsTrue(observedAdvance); Assert.IsTrue(protocolPipe.TryMoveNext()); Assert.AreEqual(BackendType.ReadyForQuery, protocolPipe.Current.Header.Type); @@ -239,15 +239,15 @@ public void BackendMessageContext_CurrentLifetime_ExhaustiveShortSequences() { case LifetimeAction.LoadCommandComplete: case LifetimeAction.LoadReadyForQuery: - // ProtocolReadPipe retires the prior batch before committing replacement - // storage. Model that ownership boundary rather than calling SetBatch as a + // ProtocolReadPipe retires the prior cursor before committing replacement + // storage. Model that ownership boundary rather than calling SetCursor as a // replacement operation it is not. - context.RetireCurrentBatch(); + context.RetireCursor(); current = null; loaded = action is LifetimeAction.LoadCommandComplete ? BackendType.CommandComplete : BackendType.ReadyForQuery; - context.SetBatch(new(new ReadOnlySequence( + context.SetCursor(new(new ReadOnlySequence( BackendMessageBytes(loaded.Value, 6)))); break; case LifetimeAction.MoveNext: @@ -259,7 +259,7 @@ public void BackendMessageContext_CurrentLifetime_ExhaustiveShortSequences() } break; case LifetimeAction.Retire: - context.RetireCurrentBatch(); + context.RetireCursor(); loaded = null; current = null; break; @@ -298,7 +298,7 @@ public async Task RepeatedQueryFrames_WithSmallRecycledBuffers_NeverEnterMessage new StreamPipeReaderOptions(bufferSize: 1024, useZeroByteReads: false), supportCancelPending: false); var readPipe = new ProtocolReadPipe(reader, - BackendMessageBatch.DefaultDataRowStreamingThreshold, + BackendMessageCursor.DefaultDataRowStreamingThreshold, ownsReader: true); var messageIndex = 0; while (await readPipe.MoveNextAsync(default)) @@ -338,13 +338,13 @@ public async Task RepeatedQueryFrames_ThroughDirectReads_NeverEnterMessageBodies new StreamPipeReaderOptions(bufferSize: 1024, useZeroByteReads: false), supportCancelPending: false); var readPipe = new ProtocolReadPipe(reader, - BackendMessageBatch.DefaultDataRowStreamingThreshold, + BackendMessageCursor.DefaultDataRowStreamingThreshold, ownsReader: true); var directReader = (StreamPipeReader)readPipe.PipeReader; var messageIndex = 0; while (true) { - readPipe.PrepareMoveNextBatch(); + readPipe.PrepareRead(); Assert.IsTrue(directReader.SupportsDirectRead); var read = directReader.BeginDirectRead(default); while (true) @@ -354,10 +354,10 @@ public async Task RepeatedQueryFrames_ThroughDirectReads_NeverEnterMessageBodies { continue; } - if (readPipe.CompleteMoveNextBatch( + if (readPipe.CompleteRead( result, default, out var completed)) { - ValidateBatch(); + ValidateMessages(); break; } if (completed) @@ -370,7 +370,7 @@ public async Task RepeatedQueryFrames_ThroughDirectReads_NeverEnterMessageBodies Assert.AreEqual(repetitions * response.Length, messageIndex); await readPipe.DisposeAsync(); - void ValidateBatch() + void ValidateMessages() { while (readPipe.TryMoveNext()) { @@ -409,7 +409,7 @@ public async Task Eof_InvalidatesPublishedBackendMessage() { var pipe = new Pipe(); var readPipe = new ProtocolReadPipe(pipe.Reader, - BackendMessageBatch.DefaultDataRowStreamingThreshold); + BackendMessageCursor.DefaultDataRowStreamingThreshold); await pipe.Writer.WriteAsync(BackendMessageBytes(BackendType.ReadyForQuery, 6)); Assert.IsTrue(await readPipe.MoveNextAsync(CancellationToken.None)); @@ -438,7 +438,7 @@ public async Task BackendBodyReader_ExtendsPrefixThenSlides() decoder.Pipe.BindDecoder(decoder); await pipe.Writer.WriteAsync(wire.AsMemory(0, 8)); - Assert.IsTrue(await MoveNextBatchAsync(decoder.Pipe)); + Assert.IsTrue(await ReadNextAsync(decoder.Pipe)); Assert.IsTrue(decoder.Pipe.TryMoveNext()); var body = decoder.Pipe.Current.OpenBodyReader(); Assert.AreEqual(3, body.Buffer.Length); @@ -473,12 +473,12 @@ public async Task BackendReadPipe_ExtendedRowAdvancesToTrailingMessage() pauseWriterThreshold: 256 * 1024, resumeWriterThreshold: 128 * 1024)); var decoder = new PgDecoder(pipe.Reader, - BackendMessageBatch.DefaultDataRowStreamingThreshold, + BackendMessageCursor.DefaultDataRowStreamingThreshold, CancellationToken.None, Timeout.InfiniteTimeSpan); decoder.Pipe.BindDecoder(decoder); var initialLength = bind.Length - + BackendMessageBatch.DefaultDataRowStreamingThreshold; + + BackendMessageCursor.DefaultDataRowStreamingThreshold; await pipe.Writer.WriteAsync(wire.AsMemory(0, initialLength)); Assert.IsTrue(await decoder.Pipe.MoveNextAsync(default)); Assert.IsTrue(decoder.Pipe.TryMoveNext()); @@ -509,44 +509,44 @@ public void BackendMessage_BufferedRequiresTagAndDeclaredLength() } [TestMethod] - public void BackendBatch_WaitsForUsefulPartialDataRowPrefix() + public void BackendCursor_WaitsForUsefulPartialDataRowPrefix() { var rowLength = 128 * 1024; var wire = BackendMessageBytes(BackendType.DataRow, rowLength); var smallPrefix = new ReadOnlySequence(wire.AsMemory(0, 32)); - var batch = new BackendMessageBatch(smallPrefix); - Assert.IsFalse(batch.TryReadNextInPlace(out _, out _, out _)); - Assert.AreEqual(BackendMessageBatch.DefaultDataRowStreamingThreshold, - batch.RequiredBufferedLength); + var cursor = new BackendMessageCursor(smallPrefix); + Assert.IsFalse(cursor.TryReadNextInPlace(out _, out _, out _)); + Assert.AreEqual(BackendMessageCursor.DefaultDataRowStreamingThreshold, + cursor.RequiredBufferedLength); var usefulPrefix = new ReadOnlySequence( - wire.AsMemory(0, BackendMessageBatch.DefaultDataRowStreamingThreshold)); - batch = new(usefulPrefix); - Assert.IsTrue(batch.TryReadNextInPlace(out var rowHeader, out var partialRow, out _)); + wire.AsMemory(0, BackendMessageCursor.DefaultDataRowStreamingThreshold)); + cursor = new(usefulPrefix); + Assert.IsTrue(cursor.TryReadNextInPlace(out var rowHeader, out var partialRow, out _)); Assert.AreEqual(BackendType.DataRow, rowHeader.Type); - Assert.AreEqual(BackendMessageBatch.DefaultDataRowStreamingThreshold, partialRow.Length); + Assert.AreEqual(BackendMessageCursor.DefaultDataRowStreamingThreshold, partialRow.Length); Assert.IsFalse(new BackendMessage(rowHeader, partialRow, new BackendMessageContext(), 0).Buffered); } [TestMethod] - public void BackendBatch_FramesUnknownMessageType() + public void BackendCursor_FramesUnknownMessageType() { var wire = BackendHeaderBytes((BackendType)(byte)'o', 4); - var batch = new BackendMessageBatch(new ReadOnlySequence(wire)); + var cursor = new BackendMessageCursor(new ReadOnlySequence(wire)); - Assert.IsTrue(batch.TryReadNextInPlace(out var header, out _, out _)); + Assert.IsTrue(cursor.TryReadNextInPlace(out var header, out _, out _)); Assert.AreEqual((BackendType)(byte)'o', header.Type); } [TestMethod] - public void BackendBatch_RejectsMessageBeyondPostgreSqlAllocationLimit() + public void BackendCursor_RejectsMessageBeyondPostgreSqlAllocationLimit() { var wire = BackendHeaderBytes(BackendType.DataRow, 0x3FFF_FFFF); - var batch = new BackendMessageBatch(new ReadOnlySequence(wire)); + var cursor = new BackendMessageCursor(new ReadOnlySequence(wire)); Assert.ThrowsExactly(() => - batch.TryReadNextInPlace(out _, out _, out _)); + cursor.TryReadNextInPlace(out _, out _, out _)); } static byte[] BackendHeaderBytes(BackendType type, int length) diff --git a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs index 2a2c939..c9af025 100644 --- a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs +++ b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs @@ -143,7 +143,7 @@ public async Task GracefulCloseMidRowEofSurfacesClosedException() for (var i = 0; i < dataRowIndex; i++) transport.ReleaseSegment(messages[i]); transport.ReleaseSegment(messages[dataRowIndex] - .AsSpan(0, BackendMessageBatch.DefaultDataRowStreamingThreshold).ToArray()); + .AsSpan(0, BackendMessageCursor.DefaultDataRowStreamingThreshold).ToArray()); Assert.IsTrue(await resultPending); var rows = flowEnumerator.Current.GetAsyncEnumerator(CommandResult.RowBuffering.Streaming); diff --git a/Slon/Pg/PgClientOptions.cs b/Slon/Pg/PgClientOptions.cs index 5ed135e..eb2b022 100644 --- a/Slon/Pg/PgClientOptions.cs +++ b/Slon/Pg/PgClientOptions.cs @@ -43,7 +43,7 @@ public PostgreSqlSslOptions Ssl public TimeSpan ConnectionTimeout { get; init; } = Timeout.InfiniteTimeSpan; internal PgSessionResetOptions SessionReset { get; init; } = new(); - internal int DataRowStreamingThreshold { get; init; } = BackendMessageBatch.DefaultDataRowStreamingThreshold; + internal int DataRowStreamingThreshold { get; init; } = BackendMessageCursor.DefaultDataRowStreamingThreshold; internal int MaxInFlightFlowsPerWire { get; init; } internal PipelineScheduler? ExecutionScheduler { get; init; } diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index a63f7a3..bec6a63 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -9,8 +9,8 @@ namespace Slon.Pg.Protocol; sealed class BackendMessageContext { PgDecoder _decoder = null!; - BackendMessageBatch _remainingBatch; - bool _hasBatch; + BackendMessageCursor _cursor; + bool _hasCursor; BackendMessage _current; FallbackBuffer _currentFallbackBuffer; short _version; @@ -116,7 +116,7 @@ public long GetCurrentMessageOffset(short token) Validate(token); if ((_messageState & MessageOffsetCaptured) == 0) { - _currentMessageOffset = _remainingBatch.GetCurrentMessageOffset( + _currentMessageOffset = _cursor.GetCurrentMessageOffset( _current.BufferedLength); _messageState |= MessageOffsetCaptured; } @@ -185,7 +185,7 @@ CurrentMessageBuffer GetBodyBuffer(short token, CurrentMessageBuffer result) BackendMessage.Initialize( ref _current, _current.Header, message, this, token, buffered: true); var messageEnd = _currentMessageOffset + messageLength; - _remainingBatch = new BackendMessageBatch(result.Buffer).Slice(messageEnd); + _cursor = new BackendMessageCursor(result.Buffer).Slice(messageEnd); } return new(body, result.IsComplete); } @@ -274,12 +274,12 @@ public bool TryMoveNext() _publicationState = PublicationState.Current; return true; } - if (!_remainingBatch.TryReadNextInPlace(out var header, out var buffer, out var bufferLength)) + if (!_cursor.TryReadNextInPlace(out var header, out var buffer, out var bufferLength)) return false; ResetMessageState(); if (bufferLength < header.MessageLength) _decoder.SetCurrentMessageLength( - _remainingBatch.ConsumedLength - bufferLength + _cursor.ConsumedLength - bufferLength + header.MessageLength); BackendMessage.Initialize(ref _current, header, buffer, this, ++_version, bufferLength >= header.MessageLength); @@ -292,39 +292,39 @@ void ResetMessageState() } } - public void RetireCurrentBatch() + public void RetireCursor() { - // Moving the batch enumerator may return or refill the memory backing every view held here. + // Advancing the read grant may return or refill the memory backing every view held here. // A failed message poll preserves Current, but crossing this ownership boundary cannot. var invalidateToken = _publicationState is not PublicationState.None; _current = default; _currentFallbackBuffer.Clear(); _publicationState = PublicationState.None; - _remainingBatch = default; - _hasBatch = false; + _cursor = default; + _hasCursor = false; _currentMessageOffset = 0; _messageState = 0; if (invalidateToken) _version++; } - public bool TryGetBatchReadRequirement( + public bool TryGetReadRequirement( out SequencePosition consumed, out long requiredLength) { - if (!_hasBatch || _remainingBatch.RequiredBufferedLength <= 0) + if (!_hasCursor || _cursor.RequiredBufferedLength <= 0) { consumed = default; requiredLength = 0; return false; } - consumed = _remainingBatch.UnreadStart; - requiredLength = _remainingBatch.RequiredBufferedLength - - _remainingBatch.ConsumedLength; + consumed = _cursor.UnreadStart; + requiredLength = _cursor.RequiredBufferedLength + - _cursor.ConsumedLength; return true; } - // Reads the next message WITHOUT publishing it as Current. The remaining batch cursor + // Reads the next message WITHOUT publishing it as Current. The message cursor // really advances past the header, but the parsed (header, buffer) lands in the peek // slot and the follow-up TryMoveNext picks it up without re-parsing. The returned // BackendMessage is valid until the next TryMoveNext (which bumps the version token); @@ -336,14 +336,14 @@ public bool TryPeekNext(out BackendHeader header) header = _current.Header; return true; } - if (!_remainingBatch.TryReadNextInPlace( + if (!_cursor.TryReadNextInPlace( out header, out var buffer, out var bufferLength)) { return false; } if (bufferLength < header.MessageLength) _decoder.SetCurrentMessageLength( - _remainingBatch.ConsumedLength - bufferLength + _cursor.ConsumedLength - bufferLength + header.MessageLength); _messageState = 0; BackendMessage.Initialize(ref _current, header, buffer, this, ++_version, @@ -362,13 +362,13 @@ public BackendMessage Peeked } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetBatch(BackendMessageBatch batch) + public void SetCursor(BackendMessageCursor cursor) { Debug.Assert(_publicationState is PublicationState.None, - "The prior batch must be retired before publishing replacement storage."); + "The prior cursor must be retired before publishing replacement storage."); _publicationState = PublicationState.None; - _remainingBatch = batch; - _hasBatch = true; + _cursor = cursor; + _hasCursor = true; } } diff --git a/Slon/Pg/Protocol/BackendMessageBatch.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs similarity index 96% rename from Slon/Pg/Protocol/BackendMessageBatch.cs rename to Slon/Pg/Protocol/BackendMessageCursor.cs index 3b374d9..f59f029 100644 --- a/Slon/Pg/Protocol/BackendMessageBatch.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -6,8 +6,8 @@ namespace Slon.Pg.Protocol; -// Note: both the batch and the segmenter are perf sensitive. -struct BackendMessageBatch(ReadOnlySequence buffer) +// The cursor is perf sensitive. +struct BackendMessageCursor(ReadOnlySequence buffer) { public const int DefaultDataRowStreamingThreshold = 16 * 1024; const uint MaxMessageLength = 0x3FFF_FFFF; @@ -17,11 +17,11 @@ struct BackendMessageBatch(ReadOnlySequence buffer) readonly int _dataRowStreamingThreshold = DefaultDataRowStreamingThreshold; long _requiredBufferedLength; - internal BackendMessageBatch( + internal BackendMessageCursor( ReadOnlySequence buffer, int dataRowStreamingThreshold) : this(buffer) => _dataRowStreamingThreshold = dataRowStreamingThreshold; - BackendMessageBatch(ReadOnlySequence buffer, + BackendMessageCursor(ReadOnlySequence buffer, int dataRowStreamingThreshold, long initialLength) : this(buffer, dataRowStreamingThreshold) => _initialLength = initialLength; @@ -33,7 +33,7 @@ internal BackendMessageBatch( public readonly long GetCurrentMessageOffset(long currentBufferedLength) => _initialLength - _buffer.Length - currentBufferedLength; - public readonly BackendMessageBatch Slice(long offset) + public readonly BackendMessageCursor Slice(long offset) { return new(_buffer.Sequence.Slice(offset), _dataRowStreamingThreshold, _initialLength); @@ -74,7 +74,7 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength, out BackendMessageBatch remaining) + public readonly bool TryReadNext(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength, out BackendMessageCursor remaining) { var thisCopy = this; var success = thisCopy.TryReadNextInPlace(out header, out buffer, out bufferLength); diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index 8bdd046..6772b42 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -86,7 +86,7 @@ public PgClientProtocolOptions(PgClientOptions options) public TimeSpan CancellationTimeout { get; set; } = TimeSpan.FromSeconds(10); public TimeSpan CancellationRetryInterval { get; set; } = TimeSpan.FromSeconds(1); internal PgSessionResetOptions SessionReset { get; set; } = new(); - public int DataRowStreamingThreshold { get; set; } = BackendMessageBatch.DefaultDataRowStreamingThreshold; + public int DataRowStreamingThreshold { get; set; } = BackendMessageCursor.DefaultDataRowStreamingThreshold; public int MaxInFlightFlowsPerWire { get; set; } public ILoggerFactory LoggerFactory { get; set; } = NullLoggerFactory.Instance; // Datasource bootstrap supplies this. A standalone raw protocol can omit backend identity and diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 65269f7..b5f96ea 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -115,14 +115,14 @@ internal void SetCurrentMessageLength(long messageLength) internal void CompleteCurrentMessage() => _pipe.CompleteCurrentMessage(); - void PrepareMoveNextBatch() => _pipe.PrepareMoveNextBatch(); + void PrepareRead() => _pipe.PrepareRead(); - bool CompleteMoveNextBatch( + bool CompleteRead( in ReadResult result, CancellationToken cancellationToken, out bool completed) - => _pipe.CompleteMoveNextBatch( + => _pipe.CompleteRead( result, cancellationToken, out completed); - bool MoveNextBatch(TimeSpan timeout) + bool ReadNext(TimeSpan timeout) => _pipe.MoveNext(timeout); bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask task) @@ -146,7 +146,7 @@ bool CompleteDirectRead(int length, CancellationToken cancellationToken, return false; } readFinished = true; - return CompleteMoveNextBatch(result, cancellationToken, out completed); + return CompleteRead(result, cancellationToken, out completed); } void AbortDirectRead() => _directReader!.AbortDirectRead(); @@ -441,7 +441,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau return new(true); } - PrepareMoveNextBatch(); + PrepareRead(); var readToken = _cancellationTokenSource.Token; var frontierFlow = EnterCancellationReadFrontier(); @@ -483,7 +483,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau if (!readTask.IsCompletedSuccessfully) return MoveNextAsyncCore(readTask, null, null, cancellationToken, frontierFlow); LeaveCancellationReadFrontier(frontierFlow); - if (CompleteMoveNextBatch( + if (CompleteRead( readTask.Result, _cancellationTokenSource.Token, out var readCompleted)) continue; @@ -537,7 +537,7 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa LeaveCancellationReadFrontier(frontierFlow!); frontierFlow = null; readTask = null; - if (CompleteMoveNextBatch( + if (CompleteRead( result, _cancellationTokenSource.Token, out var readCompleted)) continue; @@ -615,7 +615,7 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa if (messageHandledTask.HasValue) continue; - PrepareMoveNextBatch(); + PrepareRead(); try { @@ -767,7 +767,7 @@ or PgTypes.BackendType.NotificationResponse return true; } - PrepareMoveNextBatch(); + PrepareRead(); break; } diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index e29b0f8..cded048 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -10,7 +10,7 @@ namespace Slon.Pg.Protocol; sealed class ProtocolReadPipe( PipeReader reader, int dataRowStreamingThreshold, bool ownsReader = false) { - enum PendingRead : byte { None, Batch, Slide, Extend } + enum PendingRead : byte { None, Messages, Slide, Extend } readonly BackendMessageContext _messageContext = new(); ReadOnlySequence _activeBuffer; @@ -18,7 +18,7 @@ enum PendingRead : byte { None, Batch, Slide, Extend } SequencePosition _retainedStart; long _currentMessageOffset; long _currentMessageLength = -1; - long _pendingBatchOffset; + long _pendingCursorOffset; long _pendingSkipLength; int _minimumReadSize; PendingRead _pendingRead; @@ -37,9 +37,9 @@ public bool TryPeekNext(out BackendHeader header) public void BindDecoder(PgDecoder decoder) => _messageContext.BindDecoder(decoder); - public void PrepareMoveNextBatch() + public void PrepareRead() { - if (_pendingRead is PendingRead.Batch) + if (_pendingRead is PendingRead.Messages) return; if (_pendingRead is not PendingRead.None) ThrowHelper.ThrowInvalidOperation( @@ -47,9 +47,9 @@ public void PrepareMoveNextBatch() if (!_hasActiveRead) { - _pendingBatchOffset = 0; + _pendingCursorOffset = 0; _minimumReadSize = BackendHeader.ByteCount; - _pendingRead = PendingRead.Batch; + _pendingRead = PendingRead.Messages; return; } @@ -59,38 +59,38 @@ public void PrepareMoveNextBatch() return; } - if (!_messageContext.TryGetBatchReadRequirement( + if (!_messageContext.TryGetReadRequirement( out var unread, out var requiredLength)) ThrowHelper.ThrowInvalidOperation( - "The current backend-message batch has not been exhausted."); + "The current backend-message cursor has not been exhausted."); - _pendingBatchOffset = 0; - _messageContext.RetireCurrentBatch(); + _pendingCursorOffset = 0; + _messageContext.RetireCursor(); reader.AdvanceTo(unread, _examined); _hasActiveRead = false; _activeBuffer = default; _currentMessageLength = -1; _currentMessageOffset = 0; _minimumReadSize = int.CreateSaturating(requiredLength); - _pendingRead = PendingRead.Batch; + _pendingRead = PendingRead.Messages; } void PrepareAfterPartialMessage() { var current = _activeBuffer.Slice(_currentMessageOffset); - _messageContext.RetireCurrentBatch(); + _messageContext.RetireCursor(); if (current.Length >= _currentMessageLength) { var unread = current.GetPosition(_currentMessageLength); reader.AdvanceTo(unread, unread); - _pendingBatchOffset = 0; + _pendingCursorOffset = 0; _minimumReadSize = BackendHeader.ByteCount; } else { _pendingSkipLength = _currentMessageLength - current.Length; reader.AdvanceTo(_activeBuffer.End, _examined); - _pendingBatchOffset = _pendingSkipLength; + _pendingCursorOffset = _pendingSkipLength; _minimumReadSize = int.CreateSaturating( _pendingSkipLength + BackendHeader.ByteCount); } @@ -99,7 +99,7 @@ void PrepareAfterPartialMessage() _activeBuffer = default; _currentMessageLength = -1; _currentMessageOffset = 0; - _pendingRead = PendingRead.Batch; + _pendingRead = PendingRead.Messages; } public ValueTask ReadAsync(CancellationToken cancellationToken) @@ -107,12 +107,12 @@ public ValueTask ReadAsync(CancellationToken cancellationToken) ? reader.ReadAtLeastAsync(_minimumReadSize, cancellationToken) : reader.ReadAsync(cancellationToken); - public bool CompleteMoveNextBatch( + public bool CompleteRead( in ReadResult result, CancellationToken cancellationToken, out bool completed) { - if (_pendingRead is not PendingRead.Batch) - ThrowHelper.ThrowInvalidOperation("No batch read is pending."); + if (_pendingRead is not PendingRead.Messages) + ThrowHelper.ThrowInvalidOperation("No protocol read is pending."); _pendingRead = PendingRead.None; _minimumReadSize = 0; if (result.IsCanceled) @@ -120,7 +120,7 @@ public bool CompleteMoveNextBatch( if (result.Buffer.IsEmpty && result.IsCompleted) { completed = true; - _messageContext.RetireCurrentBatch(); + _messageContext.RetireCursor(); return false; } if (result.Buffer.IsEmpty) @@ -144,40 +144,40 @@ public bool CompleteMoveNextBatch( reader.AdvanceTo(result.Buffer.End, result.Buffer.End); _hasActiveRead = false; _activeBuffer = default; - _pendingBatchOffset = _pendingSkipLength; + _pendingCursorOffset = _pendingSkipLength; _minimumReadSize = int.CreateSaturating( _pendingSkipLength + BackendHeader.ByteCount); - _pendingRead = PendingRead.Batch; + _pendingRead = PendingRead.Messages; completed = false; return false; } - _pendingBatchOffset = _pendingSkipLength; + _pendingCursorOffset = _pendingSkipLength; _pendingSkipLength = 0; } - _currentMessageOffset = _pendingBatchOffset; - var batchBuffer = _pendingBatchOffset is 0 + _currentMessageOffset = _pendingCursorOffset; + var cursorBuffer = _pendingCursorOffset is 0 ? result.Buffer - : result.Buffer.Slice(_pendingBatchOffset); - if (batchBuffer.IsEmpty) + : result.Buffer.Slice(_pendingCursorOffset); + if (cursorBuffer.IsEmpty) { completed = result.IsCompleted; if (completed) - _messageContext.RetireCurrentBatch(); + _messageContext.RetireCursor(); if (!completed) { reader.AdvanceTo(result.Buffer.End, result.Buffer.End); _hasActiveRead = false; _activeBuffer = default; - _pendingBatchOffset = 0; + _pendingCursorOffset = 0; _minimumReadSize = BackendHeader.ByteCount; - _pendingRead = PendingRead.Batch; + _pendingRead = PendingRead.Messages; } return false; } _currentMessageLength = -1; - var batch = new BackendMessageBatch( - batchBuffer, dataRowStreamingThreshold); - _messageContext.SetBatch(batch); + var cursor = new BackendMessageCursor( + cursorBuffer, dataRowStreamingThreshold); + _messageContext.SetCursor(cursor); completed = false; return true; } @@ -316,9 +316,9 @@ public async ValueTask MoveNextAsync( { while (true) { - PrepareMoveNextBatch(); + PrepareRead(); var read = await ReadAsync(cancellationToken).ConfigureAwait(false); - if (CompleteMoveNextBatch( + if (CompleteRead( read, cancellationToken, out var completed)) return true; if (completed) @@ -333,11 +333,11 @@ public bool MoveNext(TimeSpan timeout) "Underlying pipe reader does not support synchronous reads."); while (true) { - PrepareMoveNextBatch(); + PrepareRead(); var read = _minimumReadSize > 0 ? syncReader.ReadAtLeast(_minimumReadSize, timeout) : syncReader.Read(timeout); - if (CompleteMoveNextBatch( + if (CompleteRead( read, CancellationToken.None, out var completed)) return true; if (completed) @@ -356,14 +356,14 @@ public void CompleteCurrentMessage() public void Dispose() { - _messageContext.RetireCurrentBatch(); + _messageContext.RetireCursor(); if (ownsReader) reader.Complete(); } public ValueTask DisposeAsync() { - _messageContext.RetireCurrentBatch(); + _messageContext.RetireCursor(); return ownsReader ? reader.CompleteAsync() : default; } } diff --git a/Slon/SlonDataSourceOptions.cs b/Slon/SlonDataSourceOptions.cs index ed4a176..0f34a1f 100644 --- a/Slon/SlonDataSourceOptions.cs +++ b/Slon/SlonDataSourceOptions.cs @@ -112,7 +112,7 @@ public int PoolSize /// /// DataRows larger than this may cross the decoder boundary before their complete body has arrived. /// - public int DataRowStreamingThreshold { get; init; } = BackendMessageBatch.DefaultDataRowStreamingThreshold; + public int DataRowStreamingThreshold { get; init; } = BackendMessageCursor.DefaultDataRowStreamingThreshold; /// Configures which connection state is reset when an exclusive scope is released. internal PgSessionResetOptions SessionReset { get; init; } = new(); /// From 399590ecb8437eccdbccc5f37dbf59747f7f994d Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Tue, 1 Sep 2026 22:39:28 +0200 Subject: [PATCH 012/136] Retain result-set memory in the protocol read pipe --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 24 ++++ .../Pg/CommandResultEnumerationTests.cs | 106 ++++++++++++++++++ Slon/Pg/CommandResult.cs | 15 +++ Slon/Pg/Protocol/BackendMessage.cs | 18 +++ Slon/Pg/Protocol/BackendMessageContext.cs | 73 +++++++++++- .../Flows/CommandFlow.MessageEnumerator.cs | 15 +++ Slon/Pg/Protocol/PgClientProtocol.cs | 1 + Slon/Pg/Protocol/PgDecoder.cs | 55 ++++++++- Slon/Pg/Protocol/ProtocolReadPipe.cs | 90 ++++++++++++--- Slon/Pg/Row.cs | 36 ++++++ 10 files changed, 415 insertions(+), 18 deletions(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index eee0ec0..ca5e5e8 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Buffers.Binary; using System.IO.Pipelines; +using System.Runtime.InteropServices; using Slon.Pipelines; using Slon.Pg.Protocol; using static Slon.Pg.Protocol.PgTypes; @@ -182,6 +183,29 @@ public void BackendMessageCursor_AdvancesAcrossExactSegmentBoundary() Assert.IsFalse(cursor.TryReadNextInPlace(out _, out _, out _)); } + [TestMethod] + public void ContiguousMemory_StraddleIsProjectedOncePerResultTenure() + { + var bytes = BackendMessageBytes( + BackendType.DataRow, new byte[] { 0, 1, 2, 3, 4, 5 }); + var context = new BackendMessageContext(); + context.SetCursor(new(Segmented( + bytes.AsMemory(0, 7), bytes.AsMemory(7)))); + Assert.IsTrue(context.TryMoveNext()); + var field = context.Current.GetSequence(); + Assert.IsFalse(field.IsSingleSegment); + + var first = context.Current.GetContiguousMemory(field); + var second = context.Current.GetContiguousMemory(field); + CollectionAssert.AreEqual(field.ToArray(), first.ToArray()); + Assert.IsTrue(MemoryMarshal.TryGetArray(first, out var firstArray)); + Assert.IsTrue(MemoryMarshal.TryGetArray(second, out var secondArray)); + Assert.AreSame(firstArray.Array, secondArray.Array); + + context.ReleaseContiguousProjections(); + context.RetireCursor(); + } + [TestMethod] public async Task MovingToNextRead_RetiresCurrentBeforeReturningItsStorage() { diff --git a/Slon.Tests/Pg/CommandResultEnumerationTests.cs b/Slon.Tests/Pg/CommandResultEnumerationTests.cs index 10a52c5..c712c6c 100644 --- a/Slon.Tests/Pg/CommandResultEnumerationTests.cs +++ b/Slon.Tests/Pg/CommandResultEnumerationTests.cs @@ -1,3 +1,4 @@ +using System.Text; using Slon.Pg; using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; @@ -7,6 +8,111 @@ namespace Slon.Tests.Pg; [TestClass] public class CommandResultEnumerationTests { + [ConnectionCreatingTestMethod] + public async Task ResultSetBuffering_CannotBeginAfterRowEnumeration() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(new CommandFlow( + async: true, Command.Create("select generate_series(1, 2)"))); + var results = flow.GetAsyncEnumerator(); + + Assert.IsTrue(await results.MoveNextAsync()); + var result = results.Current; + var rows = result.GetAsyncEnumerator(); + Assert.IsTrue(await rows.MoveNextAsync()); + Assert.ThrowsExactly( + result.EnableResultSetBuffering); + + await rows.DisposeAsync(); + await results.DisposeAsync(); + } + + [ConnectionCreatingTestMethod] + public async Task ContiguousFieldMemory_RemainsValidAcrossExtendedBatches() + { + const int rowCount = 2000; + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(new CommandFlow(async: true, Command.Create( + $"select i, i::text || repeat('x', 96) from generate_series(1, {rowCount}) as i"))); + var results = flow.GetAsyncEnumerator(); + var values = new List<(int Id, ReadOnlyMemory Message)>(rowCount); + + try + { + Assert.IsTrue(await results.MoveNextAsync()); + results.Current.EnableResultSetBuffering(); + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync()) + { + var reader = rows.Current.GetReader(); + values.Add((reader.Read(), reader.ReadBorrowedMemory())); + } + await rows.DisposeAsync(); + + Assert.AreEqual(rowCount, values.Count); + foreach (var (id, message) in values) + Assert.AreEqual(id + new string('x', 96), + Encoding.UTF8.GetString(message.Span)); + } + finally + { + await results.DisposeAsync(); + } + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task ContiguousFieldMemory_BuffersStreamingRowsIntoTheRetainedBatch() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(new CommandFlow(async: true, Command.Create( + "select i, i::text || repeat('x', 20000) from generate_series(1, 3) as i"))); + var results = flow.GetAsyncEnumerator(); + var values = new List>(); + + try + { + Assert.IsTrue(await results.MoveNextAsync()); + results.Current.EnableResultSetBuffering(); + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync()) + values.Add(rows.Current.BorrowFieldMemory(1)); + await rows.DisposeAsync(); + + Assert.AreEqual(3, values.Count); + for (var i = 0; i < values.Count; i++) + Assert.AreEqual((i + 1) + new string('x', 20000), + Encoding.UTF8.GetString(values[i].Span)); + } + finally + { + await results.DisposeAsync(); + } + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task ResultSetBuffering_AbandonmentReleasesTheReadGrant() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(new CommandFlow(async: true, Command.Create( + "select i, i::text || repeat('x', 96) from generate_series(1, 2000) as i"))); + var results = flow.GetAsyncEnumerator(); + + Assert.IsTrue(await results.MoveNextAsync()); + results.Current.EnableResultSetBuffering(); + var rows = results.Current.GetAsyncEnumerator(); + Assert.IsTrue(await rows.MoveNextAsync()); + var borrowed = rows.Current.BorrowFieldMemory(1); + Assert.IsFalse(borrowed.IsEmpty); + + await rows.DisposeAsync(); + await results.DisposeAsync(); + await PgTestPool.RunAsync(protocol, "select 1"); + } + [ConnectionCreatingTestMethod] public async Task DescribeOnlyErrorSurfacesWhenInspectingTheResult() { diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index f4ba2f8..4a81211 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -101,6 +101,21 @@ public RowEnumerator GetAsyncEnumerator(RowBuffering buffering, CancellationToke return new(this, buffering); } + /// + /// Retains the memory backing this result set while its rows are enumerated. + /// + /// + /// Retention may cause subsequent rows to be buffered. Memory returned by + /// remains valid until this command result is released. + /// + public void EnableResultSetBuffering() + { + if (_firstRowEnumerated) + ThrowHelper.ThrowInvalidOperation( + "Result-set buffering must be enabled before row enumeration begins."); + _messageEnumerator.EnableResultSetBuffering(); + } + public bool TryGetCommandComplete([NotNullWhen(true)]out CommandCompleteMessage? value) { // For commands without rows we enumerate once ourselves. diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index 3bf1c6e..4575f05 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -159,6 +159,24 @@ public ReadOnlySequence GetSequence(long offset) public ReadOnlySequence GetSequence() => GetSequence(0); + internal ReadOnlyMemory GetContiguousMemory( + ReadOnlyMemory source) + { + EnsureBodyWindowAvailable(); + return IsIndependent + ? source + : Context.GetContiguousMemory(Token, source); + } + + internal ReadOnlyMemory GetContiguousMemory( + in ReadOnlySequence source) + { + EnsureBodyWindowAvailable(); + if (IsIndependent) + return source.IsSingleSegment ? source.First : source.ToArray(); + return Context.GetContiguousMemory(Token, source); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGetFirstSpan(int offset, out ReadOnlySpan span) { diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index bec6a63..0697901 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -24,6 +24,7 @@ sealed class BackendMessageContext enum PublicationState : byte { None, Current, Peeked } PublicationState _publicationState; long _currentMessageOffset; + ContiguousProjection? _contiguousProjections; struct FallbackBuffer { ReadOnlySequenceSegment? _start; @@ -59,6 +60,14 @@ public readonly ReadOnlySequence Sequence => new(_start!, _startIndex, _end!, _endIndex); } + sealed class ContiguousProjection + { + public required byte[] Buffer { get; init; } + public required SequencePosition Start { get; init; } + public required int Length { get; init; } + public ContiguousProjection? Next { get; init; } + } + public BackendMessage Current { @@ -123,6 +132,55 @@ public long GetCurrentMessageOffset(short token) return _currentMessageOffset; } + public ReadOnlyMemory GetContiguousMemory( + short token, ReadOnlyMemory source) + { + Validate(token); + return source; + } + + public ReadOnlyMemory GetContiguousMemory( + short token, in ReadOnlySequence source) + { + Validate(token); + if (source.IsSingleSegment) + return source.First; + if (source.Length > int.MaxValue) + throw new ArgumentOutOfRangeException(nameof(source)); + + var length = (int)source.Length; + for (var projection = _contiguousProjections; + projection is not null; + projection = projection.Next) + { + if (projection.Start.Equals(source.Start) + && projection.Length == length) + return projection.Buffer.AsMemory(0, length); + } + + var buffer = ArrayPool.Shared.Rent(length); + source.CopyTo(buffer); + _contiguousProjections = new() + { + Buffer = buffer, + Start = source.Start, + Length = length, + Next = _contiguousProjections + }; + return buffer.AsMemory(0, length); + } + + public void ReleaseContiguousProjections() + { + var projection = _contiguousProjections; + _contiguousProjections = null; + while (projection is not null) + { + ArrayPool.Shared.Return(projection.Buffer); + projection = projection.Next; + } + } + public void BindDecoder(PgDecoder decoder) { if (!ReferenceEquals(_decoder, decoder)) @@ -292,8 +350,10 @@ void ResetMessageState() } } - public void RetireCursor() + public void RetireCursor(bool retainProjections = false) { + if (!retainProjections) + ReleaseContiguousProjections(); // Advancing the read grant may return or refill the memory backing every view held here. // A failed message poll preserves Current, but crossing this ownership boundary cannot. var invalidateToken = _publicationState is not PublicationState.None; @@ -324,6 +384,17 @@ public bool TryGetReadRequirement( return true; } + public bool TryGetCursorUnread(out SequencePosition unread) + { + if (!_hasCursor) + { + unread = default; + return false; + } + unread = _cursor.UnreadStart; + return true; + } + // Reads the next message WITHOUT publishing it as Current. The message cursor // really advances past the header, but the parsed (header, buffer) lands in the peek // slot and the follow-up TryMoveNext picks it up without re-parsing. The returned diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 3dc0eca..7b799c9 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -61,6 +61,9 @@ public void Initialize(in Command command, PgDecoder decoder) public void Reset() => _messageEnumerator.Reset(); + public void EnableResultSetBuffering() + => _messageEnumerator.EnableResultSetBuffering(); + public (PgError Error, TransactionStatus TransactionStatus)? CompleteError => _messageEnumerator.CompleteError; @@ -283,6 +286,8 @@ async ValueTask DrainRowsAndComplete(PgDecoder decoder) public void Initialize(in Command command, PgDecoder decoder) { + if (_decoder is not null) + _decoder.ResultSetBuffering = false; _describeOnly = command.DescribeOnly; _withSync = command.WithSync; if (!ReferenceEquals(_decoder, decoder)) @@ -299,6 +304,8 @@ public void Initialize(in Command command, PgDecoder decoder) public void Reset() { + if (_decoder is not null) + _decoder.ResultSetBuffering = false; _describeOnly = false; _withSync = false; _decoder = null!; @@ -309,6 +316,14 @@ public void Reset() _done = true; } + public void EnableResultSetBuffering() + { + if (_disposed) + ThrowHelper.ThrowInvalidOperation( + "The command result has already been released."); + _decoder.ResultSetBuffering = true; + } + public (PgError Error, TransactionStatus TransactionStatus)? CompleteError { get diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index 6772b42..d412457 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1832,6 +1832,7 @@ internal void AssignCancellationBoundary(PgClientFlow flow, int window) internal void OnReleasing(PgClientFlow flow) { + Decoder.EndResultSetBuffering(flow); protocol._serverParameterState.CommitFlow(); ClearCancellationActivation(flow); var idle = ActivatedFlow is null; diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index b5f96ea..2d5cf07 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -33,6 +33,7 @@ public sealed class PgDecoder: IEnumerator, IAsyncEnumerator _pipe.CompleteCurrentMessage(); - void PrepareRead() => _pipe.PrepareRead(); + internal bool ResultSetBuffering + { + get => _resultSetBufferingOwner is not null; + set + { + if (value) + { + var owner = CurrentExecutionControl.Flow; + if (!ReferenceEquals(_resultSetBufferingOwner, owner)) + _resultSetBufferingOwner = owner; + _pipe.EnableResultSetRetention(); + } + else + { + if (_resultSetBufferingOwner is null) + return; + _resultSetBufferingOwner = null; + _pipe.EndResultSetRetention(); + } + } + } + + internal void EndResultSetBuffering(PgClientFlow owner) + { + if (!ReferenceEquals(_resultSetBufferingOwner, owner)) + return; + _resultSetBufferingOwner = null; + _pipe.EndResultSetRetention(); + } + + void ValidateResultSetBufferingOwner() + { + var owner = _resultSetBufferingOwner; + if (owner is not null + && !ReferenceEquals(CurrentExecutionControl.Flow, owner)) + EndResultSetBuffering(owner); + } + + void PrepareRead() + { + ValidateResultSetBufferingOwner(); + _pipe.PrepareRead(); + } bool CompleteRead( in ReadResult result, CancellationToken cancellationToken, out bool completed) - => _pipe.CompleteRead( + { + ValidateResultSetBufferingOwner(); + return _pipe.CompleteRead( result, cancellationToken, out completed); + } bool ReadNext(TimeSpan timeout) - => _pipe.MoveNext(timeout); + { + ValidateResultSetBufferingOwner(); + return _pipe.MoveNext(timeout); + } bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask task) { diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index cded048..dcb27fe 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -23,6 +23,7 @@ enum PendingRead : byte { None, Messages, Slide, Extend } int _minimumReadSize; PendingRead _pendingRead; bool _hasActiveRead; + bool _retainsResultSet; public PipeReader PipeReader => reader; public BackendMessage Current => _messageContext.Current; @@ -45,6 +46,8 @@ public void PrepareRead() ThrowHelper.ThrowInvalidOperation( "The current message still has a pending read."); + var retainsResultSet = _retainsResultSet; + if (!_hasActiveRead) { _pendingCursorOffset = 0; @@ -55,7 +58,7 @@ public void PrepareRead() if (_currentMessageLength > 0) { - PrepareAfterPartialMessage(); + PrepareAfterPartialMessage(retainsResultSet); return; } @@ -64,9 +67,13 @@ public void PrepareRead() ThrowHelper.ThrowInvalidOperation( "The current backend-message cursor has not been exhausted."); - _pendingCursorOffset = 0; - _messageContext.RetireCursor(); - reader.AdvanceTo(unread, _examined); + _pendingCursorOffset = retainsResultSet + ? _activeBuffer.Slice(0, unread).Length + : 0; + _messageContext.RetireCursor( + retainProjections: retainsResultSet); + reader.AdvanceTo( + retainsResultSet ? _retainedStart : unread, _examined); _hasActiveRead = false; _activeBuffer = default; _currentMessageLength = -1; @@ -75,11 +82,23 @@ public void PrepareRead() _pendingRead = PendingRead.Messages; } - void PrepareAfterPartialMessage() + void PrepareAfterPartialMessage(bool retainsResultSet) { var current = _activeBuffer.Slice(_currentMessageOffset); - _messageContext.RetireCursor(); - if (current.Length >= _currentMessageLength) + _messageContext.RetireCursor( + retainProjections: retainsResultSet); + if (retainsResultSet) + { + _pendingCursorOffset = checked( + _currentMessageOffset + _currentMessageLength); + var examined = current.Length >= _currentMessageLength + ? current.GetPosition(_currentMessageLength) + : _examined; + reader.AdvanceTo(_retainedStart, examined); + _minimumReadSize = int.CreateSaturating( + _pendingCursorOffset + BackendHeader.ByteCount); + } + else if (current.Length >= _currentMessageLength) { var unread = current.GetPosition(_currentMessageLength); reader.AdvanceTo(unread, unread); @@ -161,15 +180,21 @@ public bool CompleteRead( if (cursorBuffer.IsEmpty) { completed = result.IsCompleted; - if (completed) + if (completed && !_retainsResultSet) _messageContext.RetireCursor(); if (!completed) { - reader.AdvanceTo(result.Buffer.End, result.Buffer.End); + reader.AdvanceTo( + _retainsResultSet ? _retainedStart : result.Buffer.End, + result.Buffer.End); _hasActiveRead = false; _activeBuffer = default; - _pendingCursorOffset = 0; - _minimumReadSize = BackendHeader.ByteCount; + if (!_retainsResultSet) + _pendingCursorOffset = 0; + _minimumReadSize = _retainsResultSet + ? int.CreateSaturating( + _pendingCursorOffset + BackendHeader.ByteCount) + : BackendHeader.ByteCount; _pendingRead = PendingRead.Messages; } return false; @@ -260,12 +285,21 @@ void PrepareCurrentMessageRead( throw new ArgumentOutOfRangeException(nameof(consumedLength)); reader.AdvanceTo( - mode is PendingRead.Slide ? consumed : _retainedStart, + mode is PendingRead.Slide && !_retainsResultSet + ? consumed + : _retainedStart, _examined); if (mode is PendingRead.Slide) { - _retainedStart = consumed; - _currentMessageOffset = 0; + if (_retainsResultSet) + { + _currentMessageOffset = _activeBuffer.Slice(0, consumed).Length; + } + else + { + _retainedStart = consumed; + _currentMessageOffset = 0; + } _currentMessageLength -= consumedLength; } _hasActiveRead = false; @@ -354,6 +388,34 @@ public void SetCurrentMessageLength(long messageLength) public void CompleteCurrentMessage() => _currentMessageLength = -1; + public void EnableResultSetRetention() + { + if (!_hasActiveRead || _pendingRead is not PendingRead.None) + ThrowHelper.ThrowInvalidOperation( + "Result-set retention requires an active backend message."); + _ = _messageContext.Current; + _retainsResultSet = true; + } + + public void EndResultSetRetention() + { + _retainsResultSet = false; + if (!_hasActiveRead || _pendingRead is not PendingRead.None + || _currentMessageLength > 0 + || !_messageContext.TryGetCursorUnread(out var unread)) + { + _messageContext.ReleaseContiguousProjections(); + return; + } + + _messageContext.RetireCursor(); + reader.AdvanceTo(unread, _examined); + _hasActiveRead = false; + _activeBuffer = default; + _currentMessageOffset = 0; + _pendingCursorOffset = 0; + } + public void Dispose() { _messageContext.RetireCursor(); diff --git a/Slon/Pg/Row.cs b/Slon/Pg/Row.cs index 970a465..8dd1536 100644 --- a/Slon/Pg/Row.cs +++ b/Slon/Pg/Row.cs @@ -51,6 +51,32 @@ SequenceReader GetColumnReader(int ordinal, out int columnIndex, out int c public T GetValue(int ordinal) => GetValueCore(ordinal, textEncoding: null); + /// + /// Borrows the field's raw PostgreSQL representation as contiguous memory. + /// + /// + /// + /// The returned memory is a view over storage owned by the command result. Its lifetime is not + /// enforced: after the row enumerator advances or is disposed, accessing it may observe storage + /// that has been reused for unrelated data rather than throw. + /// + /// + /// Calling before row enumeration extends + /// the borrow until that command result is released. Copy the memory when it must outlive the + /// applicable boundary. + /// + /// + public ReadOnlyMemory BorrowFieldMemory(int ordinal) + { + RevokeColumnLease(); + EnsureBuffered(); + if (TryGetFieldMemory(ordinal, out var field)) + return Message.GetContiguousMemory(field); + + var sequence = GetFieldSequence(ordinal); + return Message.GetContiguousMemory(sequence); + } + // Bootstrap consumers have no serializer but must still bind text decoding to one negotiated // encoding snapshot for the lifetime of their operation. internal T GetValue(int ordinal, Encoding textEncoding) @@ -552,6 +578,16 @@ public T Read() _remaining = _remaining.Slice(sizeof(int) + length); return BootstrapFieldDecoder.Read(field); } + + /// + /// Borrows the next field's raw PostgreSQL representation as contiguous memory. + /// + /// + /// The returned memory has the same unenforced lifetime as + /// . + /// + public ReadOnlyMemory ReadBorrowedMemory() + => _row.BorrowFieldMemory(_ordinal++); } internal void Initialize(RowDescription rowDescription) From 7bbe5b10833d04789a7dfaa97f099d474b4e1031 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 00:30:30 +0200 Subject: [PATCH 013/136] Keep flow completion arbitration with its owner --- .../Protocol/Flows/CommandFlow.Enumerator.cs | 46 ++++++--- Slon/Pg/Protocol/Flows/CommandFlow.cs | 39 +++++--- .../Flows/FlowCallerInteractionCore.cs | 16 +-- Slon/Pg/Protocol/PgClientFlow.cs | 60 +++++++++--- Slon/Pg/Protocol/PgClientProtocol.cs | 2 + .../Sources/ManualResetValueTaskSourceCore.cs | 97 ++----------------- 6 files changed, 127 insertions(+), 133 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs index 52d15af..78eb24b 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs @@ -6,6 +6,7 @@ namespace Slon.Pg.Protocol.Flows; partial class CommandFlow { Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _enumeratorMoveNextTaskSource; + int _enumeratorMoveNextCompletionClaim; // Serializes move-next rearming against body termination. Otherwise Reset can replace the generation // just before terminal completion and strand the consumer (see MoveNextRearm.tla). Never hold it while // dispatching the gate, which may run the body inline. @@ -27,22 +28,44 @@ partial class CommandFlow void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _enumeratorMoveNextTaskSource.OnCompleted(continuation, state, token, flags); + bool TrySetEnumeratorResult(bool result, bool runContinuationsAsynchronously) + { + if (Interlocked.CompareExchange(ref _enumeratorMoveNextCompletionClaim, 1, 0) != 0) + return false; + _enumeratorMoveNextTaskSource.SetResult(result, runContinuationsAsynchronously); + return true; + } + + bool TrySetEnumeratorException(Exception exception, bool runContinuationsAsynchronously) + { + if (Interlocked.CompareExchange(ref _enumeratorMoveNextCompletionClaim, 1, 0) != 0) + return false; + _enumeratorMoveNextTaskSource.SetException(exception, runContinuationsAsynchronously); + return true; + } + + void ResetEnumeratorMoveNextSource() + { + _enumeratorMoveNextTaskSource.Reset(); + Volatile.Write(ref _enumeratorMoveNextCompletionClaim, 0); + } + // Consumer completion must dispatch asynchronously because it may run while the pipeline still owns // the current execution frame. - void CompleteEnumeration() + void CompleteEnumeration(bool runContinuationsAsynchronously = true) { // Drain errors outrank cancellation and clean completion. Preserve every error across a batch. if (_drainErrors is { Count: > 0 } errors) { Exception fault = errors.Count == 1 ? errors[0] : new AggregateException(errors); - _enumeratorMoveNextTaskSource.TrySetException(fault, runContinuationsAsynchronously: true); + TrySetEnumeratorException(fault, runContinuationsAsynchronously); } else if (Volatile.Read(ref _cancellationState) is { DeliverOce: true } cancellation && !_consumerDisposed) - _enumeratorMoveNextTaskSource.TrySetException( - new OperationCanceledException(cancellation.DeliverToken), runContinuationsAsynchronously: true); + TrySetEnumeratorException( + new OperationCanceledException(cancellation.DeliverToken), runContinuationsAsynchronously); else - _enumeratorMoveNextTaskSource.TrySetResult(false, runContinuationsAsynchronously: true); + TrySetEnumeratorResult(false, runContinuationsAsynchronously); // _enumeratorCompleted was set by the caller (SetResult's completed branch) before this runs. SignalPumpProgress(); } @@ -95,7 +118,7 @@ void CompleteEnumerationWithClose(Exception closeException) // terminal enumeration state when the close wins the generation; otherwise the consumer must // observe the result once, rearm, and self-deliver the latched close on its next move. _callerInteractionCore.SetCloseLatch(closeException); - if (_enumeratorMoveNextTaskSource.TrySetException(closeException, runContinuationsAsynchronously: true)) + if (TrySetEnumeratorException(closeException, runContinuationsAsynchronously: true)) PublishEnumerationCompleted(); SignalPumpProgress(); } @@ -120,21 +143,21 @@ void EnsureEnumerationCompleted() if (_enumeratorMoveNextTaskSource.GetStatus(_enumeratorMoveNextTaskSource.Version) is not ValueTaskSourceStatus.Pending) return; if (_callerInteractionCore.CloseException is { } latched) - _enumeratorMoveNextTaskSource.TrySetException(latched, runContinuationsAsynchronously: true); + TrySetEnumeratorException(latched, runContinuationsAsynchronously: true); else { var cancellation = Volatile.Read(ref _cancellationState); var effectiveCancellationToken = EffectiveCancellationToken; if (cancellation is { } && Volatile.Read(ref cancellation.Requested) || effectiveCancellationToken.IsCancellationRequested) - _enumeratorMoveNextTaskSource.TrySetException( + TrySetEnumeratorException( new OperationCanceledException(effectiveCancellationToken.IsCancellationRequested ? effectiveCancellationToken : cancellation!.DeliverToken), runContinuationsAsynchronously: true); // Rearming after a clean terminal still needs to complete the new generation. else if (IsEnumerationCompleted) - _enumeratorMoveNextTaskSource.TrySetResult(false, runContinuationsAsynchronously: true); + TrySetEnumeratorResult(false, runContinuationsAsynchronously: true); } } @@ -183,7 +206,7 @@ public bool MoveNext() // See MoveNextAsync: rearm only on a non-first call; the first-call source is fresh and the // body's first delivery lands on it. if (flow._consumerAdvanced) - flow._enumeratorMoveNextTaskSource.Reset(); + flow.ResetEnumeratorMoveNextSource(); flow._consumerAdvanced = true; } // Close-latch self-deliver (sync): under close this call completes the generation it just @@ -259,7 +282,6 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken) if (cancellationToken.CanBeCanceled) { flow.SetCallerCancellationToken(cancellationToken); - flow._enumeratorMoveNextTaskSource.CanCompleteConcurrently = true; } // Terminal enumeration state may outlive its completed source generation. Complete the @@ -281,7 +303,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken) // current generation for the body's one-shot terminal. Body-initiated drain retains a live // consumer and therefore continues rearming. if (flow._consumerAdvanced && !Volatile.Read(ref flow._consumerDisposed)) - flow._enumeratorMoveNextTaskSource.Reset(); + flow.ResetEnumeratorMoveNextSource(); flow._consumerAdvanced = true; } // Drive the body; teardown may already have faulted the gate. diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 9e14a38..c680196 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -191,7 +191,6 @@ private protected CommandFlow(bool async, TimeSpan? pendingTimeout) : this() { IsAsync = async; _pendingTimeout = pendingTimeout; - _enumeratorMoveNextTaskSource.CanCompleteConcurrently = true; } public CommandFlow Initialize(bool async, params ReadOnlySpan commands) @@ -205,8 +204,6 @@ public CommandFlow Initialize(bool async, in CommandFlowOptions options) _commands = options.Commands; _pendingTimeout = options.PendingTimeout; options.Observer?.OnStarted(this, options.ObserverState); - // Arm before publication: teardown may complete the source concurrently even before enumeration. - _enumeratorMoveNextTaskSource.CanCompleteConcurrently = true; return this; } @@ -266,9 +263,7 @@ internal override CancellationToken MigrationCancellationToken public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { - // Body, teardown, and cancellation may complete the source concurrently. A missing per-call token - // must not replace the flow token captured at submission. - _enumeratorMoveNextTaskSource.CanCompleteConcurrently = true; + // A missing per-call token must not replace the flow token captured at submission. if (cancellationToken.CanBeCanceled) GetOrCreateCancellationState().FlowToken = cancellationToken; return new(this); @@ -839,6 +834,7 @@ await _commands.ItemRef(_commandIndex) void SetResult(CommandResult? next) { var completed = next is null; + var publishAsync = IsAsync; if (completed) { _enumeratorCurrent = null; @@ -861,17 +857,31 @@ void SetResult(CommandResult? next) } if (completed) { - // Publish durable terminal state and complete the current generation atomically with - // respect to consumer rearming. Completion dispatches asynchronously, so it cannot reenter - // this lock or the pipeline frame that still owns the shared promise. + // Publish durable terminal state atomically with respect to consumer rearming. Async + // consumers complete from the protocol scheduler so they cannot reenter this lock or + // the pipeline frame that still owns the shared promise; sync consumers retain their + // caller-driven completion. using (_rearmLock.EnterScope()) { PublishEnumerationCompleted(); - CompleteEnumeration(); + if (!publishAsync) + CompleteEnumeration(); } + if (publishAsync) + context.SubmitDetached(static state => ((CommandFlow)state!) + .CompleteEnumeration(runContinuationsAsynchronously: false), this); return; } - _enumeratorMoveNextTaskSource.SetResult(true, runContinuationsAsynchronously: true); + if (publishAsync) + { + // Queue the publication itself so the body reaches its next caller gate before user code + // resumes. Routing through the protocol scheduler preserves that ordering without forcing + // every result continuation onto the ThreadPool. + context.SubmitDetached(static state => ((CommandFlow)state!) + .TrySetEnumeratorResult(true, runContinuationsAsynchronously: false), this); + } + else + TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); } async ValueTask ReadRfqAsync(PgDecoder decoder) @@ -1086,7 +1096,7 @@ void CompleteEnumerationWithException(Exception ex) return; // Teardown may race the consumer. The task source is the completion authority; // _enumeratorCompleted follows only when this call wins the current generation. - if (_enumeratorMoveNextTaskSource.TrySetException(ex, runContinuationsAsynchronously: true)) + if (TrySetEnumeratorException(ex, runContinuationsAsynchronously: true)) PublishEnumerationCompleted(); // A faulted body will not publish another continuation. SignalPumpProgress(); @@ -1182,10 +1192,7 @@ protected override void OnReset() Debug.Assert(IsPending || IsCompleted); _commandIndex = -1; _executePipelinedCore.Reset(); - _enumeratorMoveNextTaskSource.Reset(); - // Disarm while idle in the pool (no teardown can target a non-live flow). Initialize re-arms it - // before the flow is queued, so a live flow is always in concurrent-completion mode. - _enumeratorMoveNextTaskSource.CanCompleteConcurrently = false; + ResetEnumeratorMoveNextSource(); _enumeratorCurrent = default; _enumeratorCompleted = false; _isResultReady = false; diff --git a/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs b/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs index 8bf0042..7d23835 100644 --- a/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs +++ b/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs @@ -26,16 +26,17 @@ public Exception SetCloseLatch(Exception exception) // Inline completion transfers the body to a synchronous caller; asynchronous completion preserves // autonomous execution without running the body on the signaller's stack. Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _gate; - public void Initialize() - { - _gate.CanCompleteConcurrently = true; - } + int _gateCompletionClaim; + public void Initialize() => _gateCompletionClaim = 0; public ValueTask WaitForCaller(IValueTaskSource source) => new(source, _gate.Version); // The IVTS facade forwards through this gate surface. public void ResumeBody(bool runContinuationsAsynchronously) - => _gate.TrySetResult(default!, runContinuationsAsynchronously); + { + if (Interlocked.CompareExchange(ref _gateCompletionClaim, 1, 0) == 0) + _gate.SetResult(default!, runContinuationsAsynchronously); + } public System.Threading.Tasks.Sources.ValueTaskSourceStatus GateStatus(short token) => _gate.GetStatus(token); public void OnGateCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _gate.OnCompleted(continuation, state, token, flags); @@ -44,6 +45,7 @@ public TResult ConsumeGateResult(short token) { var result = _gate.GetResult(token); _gate.Reset(); + Volatile.Write(ref _gateCompletionClaim, 0); return result; } @@ -52,7 +54,8 @@ public void FaultBodyWait(Exception exception) // Latch first (monotone), then fault the gate, so a consumer observing the gate fault always // reads the latch set. var latched = SetCloseLatch(exception); - _gate.TrySetException(latched, runContinuationsAsynchronously: true); + if (Interlocked.CompareExchange(ref _gateCompletionClaim, 1, 0) == 0) + _gate.SetException(latched, runContinuationsAsynchronously: true); // The synchronous disposer may not have created its event yet. Publish a sticky progress // level so WaitForContinuation observes the close even when this Set would have been a no-op. SignalProgress(); @@ -187,6 +190,7 @@ public void Reset() _wakeRequested = false; _closeException = null; _gate.Reset(); + Volatile.Write(ref _gateCompletionClaim, 0); } public readonly struct CallerHandoffAwaitable(FieldRef> fieldRef) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 5bcd53e..341c6b6 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -167,6 +167,7 @@ void IThreadPoolWorkItem.Execute() // pattern). At most one pending waiter per tenure; post-completion awaits resolve // synchronously. Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _completionCore; + int _completionClaim; ManualResetEventSlim? _completionEvent; // 1 while a WaitForComplete token is live (set at capture, cleared after GetResult consumed the // core). Guards reuse: Reset bumps the core's version, so it must not run while this is set. @@ -174,6 +175,7 @@ void IThreadPoolWorkItem.Execute() // Activation state. Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _activationTaskSource; + int _activationClaim; CancellationTokenRegistration _activationCancellationTokenRegistration; TimeSpan _remainingActivationTimeout; bool _pendingTimeoutStarted; @@ -268,8 +270,38 @@ internal void WaitForSyncHandoff() protected PgClientFlow(bool supportsDeferredFlush = false) { _supportsDeferredFlush = supportsDeferredFlush; - _activationTaskSource.CanCompleteConcurrently = true; - _completionCore.CanCompleteConcurrently = true; + } + + bool TrySetActivationResult(PgDecoder decoder, bool runContinuationsAsynchronously) + { + if (Interlocked.CompareExchange(ref _activationClaim, 1, 0) != 0) + return false; + _activationTaskSource.SetResult(decoder, runContinuationsAsynchronously); + return true; + } + + bool TrySetActivationException(Exception exception, bool runContinuationsAsynchronously) + { + if (Interlocked.CompareExchange(ref _activationClaim, 1, 0) != 0) + return false; + _activationTaskSource.SetException(exception, runContinuationsAsynchronously); + return true; + } + + void ResetActivationSource() + { + _activationTaskSource.Reset(); + Volatile.Write(ref _activationClaim, 0); + } + + void CompleteFlow(Exception? exception) + { + if (Interlocked.CompareExchange(ref _completionClaim, 1, 0) != 0) + return; + if (exception is null) + _completionCore.SetResult(this, runContinuationsAsynchronously: true); + else + _completionCore.SetException(exception, runContinuationsAsynchronously: true); } protected void SetObserver(PgClientFlowObserver observer, object? state) @@ -374,8 +406,9 @@ public void Reset() // Version bump per tenure. Cross-tenure completer staleness rests on the done -> torn-down // -> retired layering (Complete precedes recycle), the same basis as the rest of this reset. _completionCore.Reset(); + Volatile.Write(ref _completionClaim, 0); _completionEvent?.Reset(); - _activationTaskSource.Reset(); + ResetActivationSource(); _rfqCount = 0; _cancellationWindow = 0; _lastMessageInducesRfq = false; @@ -484,6 +517,9 @@ public PgEncoder GetEncoder() internal ValueTask WaitForCancellationAttempt() => _executionControl.WaitForCancellationAttempt(); + internal void SubmitDetached(Action action, object? state, bool preferLocal = true) + => _executionControl.SubmitDetached(action, state, preferLocal); + internal void RequestBackendCancellation(PgClientFlow instigator, int window, BackendCancellationTiming timing, TaskCompletionSource? delivery, object episodeKey, int scope, BackendCancellationTiming subsequentTiming) @@ -666,6 +702,9 @@ internal readonly struct ExecutionControl(PgClientFlow flow, PgClientProtocol.Co public bool HasQueuedFlow => control.HasQueuedFlow; public bool IsInlineDrive => control.IsInlineDrive; + internal void SubmitDetached(Action action, object? state, bool preferLocal = true) + => control.SubmitDetached(action, state, preferLocal); + // Small optimization to allow us to skip the final sync message if we can piggyback on the flow's final rfq. public bool LastMessageInducesRfq => flow._lastMessageInducesRfq; @@ -848,7 +887,7 @@ public void Activate(PgDecoder decoder) { flow._activationCancellationTokenRegistration.Dispose(); // If none of the cancellations triggered, we have a problem, throw. - if (!flow._activationTaskSource.TrySetResult(decoder, runContinuationsAsynchronously: false) + if (!flow.TrySetActivationResult(decoder, runContinuationsAsynchronously: false) && !(flow._remainingActivationTimeout <= TimeSpan.Zero) && !control.AbortToken.IsCancellationRequested && !flow._activationCancellationTokenRegistration.Token.IsCancellationRequested) @@ -885,7 +924,7 @@ public bool PropagateTermination() if (control.AbortToken.IsCancellationRequested && !flow._completed) { var ex = control.FlowTerminationException; - flow._activationTaskSource.TrySetException(ex, runContinuationsAsynchronously: true); + flow.TrySetActivationException(ex, runContinuationsAsynchronously: true); flow.OnAbort(ex); return true; } @@ -906,7 +945,7 @@ public void OnActivationHeartbeat(TimeSpan interval) if (flow._remainingActivationTimeout != Timeout.InfiniteTimeSpan && flow._remainingActivationTimeout != TimeSpan.Zero && flow._activationTaskSource.GetStatus(flow._activationTaskSource.Version) is ValueTaskSourceStatus.Pending && (flow._remainingActivationTimeout -= interval) <= TimeSpan.Zero) - flow._activationTaskSource.TrySetException(new TimeoutException("Operation timed out waiting for activation."), runContinuationsAsynchronously: true); + flow.TrySetActivationException(new TimeoutException("Operation timed out waiting for activation."), runContinuationsAsynchronously: true); } /// Fail a never-started flow drained from the backlog at shutdown with the wire-death reason. The @@ -948,7 +987,7 @@ public void DetachForMigration(PgClientFlowSource source) if (IsDecoderSettled) { Debug.Assert(control.AbortToken.IsCancellationRequested); - flow._activationTaskSource.Reset(); + flow.ResetActivationSource(); } if (flow.HandoffEvent?.PlacementSource is not null) flow.DetachPlacementSource(source.SourceState); @@ -996,10 +1035,7 @@ public void Release(Exception? exception = null) // Async continuation dispatch: completers run in retirement/teardown contexts where // inline caller continuations are a re-entrancy hazard, the contract the old TCS's // RunContinuationsAsynchronously carried, minus its unconditional thread-pool destination. - if (exception is not null) - flow._completionCore.TrySetException(exception, runContinuationsAsynchronously: true); - else - flow._completionCore.TrySetResult(flow, runContinuationsAsynchronously: true); + flow.CompleteFlow(exception); flow._completionEvent?.Set(); // The completed observer runs from CompleteItem in the advancer/retirement work-item // context: a raw throw would crash that thread unobserved. Don't swallow either - a @@ -1075,7 +1111,7 @@ public void RegisterActivationCancellation(CancellationToken cancellationToken) ThrowHelper.ThrowInvalidOperation("Concurrent activation result awaits are not supported."); flow._activationCancellationTokenRegistration = cancellationToken.UnsafeRegister( static (state, token) => - ((PgClientFlow)state!)._activationTaskSource.TrySetException(new OperationCanceledException(token), runContinuationsAsynchronously: true), + ((PgClientFlow)state!).TrySetActivationException(new OperationCanceledException(token), runContinuationsAsynchronously: true), flow); } } diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index d412457..853c1ed 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1524,6 +1524,8 @@ public void BindPipeline( public bool IsInlineDrive => _source.IsInlineDrive; public long UnflushedBytes => protocol.UnflushedBytes; public ValueTask FlushAsync(CancellationToken cancellationToken) => protocol.FlushAsync(cancellationToken); + internal void SubmitDetached(Action action, object? state, bool preferLocal = true) + => protocol._activationScheduler.SubmitDetached(action, state, preferLocal); PgClientFlow? _cancellationActivatedFlow; internal (PgClientFlow? Owner, int Window) CancellationActivation { diff --git a/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs b/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs index 4b0e11c..ddd2004 100644 --- a/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs +++ b/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs @@ -37,18 +37,8 @@ struct ManualResetValueTaskSourceCore TResult? _result; /// The current version of this value, used to help prevent misuse. short _version; - /// Whether the current operation has completed (this can mean it's still completing, which is why _continuation is also checked). + /// Whether the current operation has completed. bool _completed; - /// Whether concurrent completions are handled correctly. - bool _canCompleteConcurrently; - - /// Gets or sets whether concurrent completions are handled correctly. - /// Enabling this allows the use of TrySet methods and makes the Set methods thread safe. - public bool CanCompleteConcurrently - { - get => _canCompleteConcurrently; - set => _canCompleteConcurrently = value; - } /// Resets to prepare for the next operation. public void Reset() @@ -60,8 +50,7 @@ public void Reset() _capturedContext = null; _error = null; _result = default; - // Release, written last: a concurrent completer's CAS (acquire) observing false then sees - // the resets above. Without it a racing TrySet could complete a half-reset source. + // Release, written last so the next tenure observes all reset state. Volatile.Write(ref _completed, false); } @@ -69,12 +58,10 @@ public void Reset() /// The result. public void SetResult(TResult result) { - var canCompleteConcurrently = _canCompleteConcurrently; - if (_completed || (canCompleteConcurrently && Interlocked.CompareExchange(ref _completed, true, false) is not false)) + if (_completed) ThrowInvalidOperationException(); - if (!canCompleteConcurrently) - _completed = true; _result = result; + Volatile.Write(ref _completed, true); new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) .SignalCompletion(runContinuationsAsynchronously: false); } @@ -84,12 +71,10 @@ public void SetResult(TResult result) /// whether to force continuations to run asynchronously this call. public void SetResult(TResult result, bool runContinuationsAsynchronously) { - var canCompleteConcurrently = _canCompleteConcurrently; - if (_completed || (canCompleteConcurrently && Interlocked.CompareExchange(ref _completed, true, false) is not false)) + if (_completed) ThrowInvalidOperationException(); - if (!canCompleteConcurrently) - _completed = true; _result = result; + Volatile.Write(ref _completed, true); new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) .SignalCompletion(runContinuationsAsynchronously); } @@ -98,12 +83,10 @@ public void SetResult(TResult result, bool runContinuationsAsynchronously) /// The exception. public void SetException(Exception error) { - var canCompleteConcurrently = _canCompleteConcurrently; - if (_completed || (canCompleteConcurrently && Interlocked.CompareExchange(ref _completed, true, false) is not false)) + if (_completed) ThrowInvalidOperationException(); - if (!canCompleteConcurrently) - _completed = true; _error = ExceptionDispatchInfo.Capture(error); + Volatile.Write(ref _completed, true); new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) .SignalCompletion(false); } @@ -113,72 +96,12 @@ public void SetException(Exception error) /// whether to force continuations to run asynchronously this call. public void SetException(Exception error, bool runContinuationsAsynchronously) { - var canCompleteConcurrently = _canCompleteConcurrently; - if (_completed || (canCompleteConcurrently && Interlocked.CompareExchange(ref _completed, true, false) is not false)) - ThrowInvalidOperationException(); - if (!canCompleteConcurrently) - _completed = true; - _error = ExceptionDispatchInfo.Capture(error); - new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) - .SignalCompletion(runContinuationsAsynchronously); - } - - /// Completes with a successful result. - /// The result. - public bool TrySetResult(TResult result) - { - if (!_canCompleteConcurrently) - ThrowInvalidOperationException(); - if (_completed || Interlocked.CompareExchange(ref _completed, true, false) is not false) - return false; - _result = result; - new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) - .SignalCompletion(false); - return true; - } - - /// Completes with a successful result. - /// The result. - /// whether to force continuations to run asynchronously this call. - public bool TrySetResult(TResult result, bool runContinuationsAsynchronously) - { - if (!_canCompleteConcurrently) - ThrowInvalidOperationException(); - if (_completed || Interlocked.CompareExchange(ref _completed, true, false) is not false) - return false; - _result = result; - new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) - .SignalCompletion(runContinuationsAsynchronously); - return true; - } - - /// Completes with an error. - /// The exception. - public bool TrySetException(Exception error) - { - if (!_canCompleteConcurrently) - ThrowInvalidOperationException(); - if (_completed || Interlocked.CompareExchange(ref _completed, true, false) is not false) - return false; - _error = ExceptionDispatchInfo.Capture(error); - new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) - .SignalCompletion(false); - return true; - } - - /// Completes with an error. - /// The exception. - /// whether to force continuations to run asynchronously this call. - public bool TrySetException(Exception error, bool runContinuationsAsynchronously) - { - if (!_canCompleteConcurrently) + if (_completed) ThrowInvalidOperationException(); - if (_completed || Interlocked.CompareExchange(ref _completed, true, false) is not false) - return false; _error = ExceptionDispatchInfo.Capture(error); + Volatile.Write(ref _completed, true); new ContinuationDispatcher(ref _continuation, ref _continuationState, ref _capturedContext) .SignalCompletion(runContinuationsAsynchronously); - return true; } /// Gets the operation version. From 717d131508743979c4e54ea09ae7309afc997fc0 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 27 Aug 2026 17:07:45 +0200 Subject: [PATCH 014/136] Decouple result parsing helpers from CommandFlow --- Slon/Pg/CommandResult.cs | 4 ++-- Slon/Pg/Protocol/Flows/CommandFlow.cs | 2 +- Slon/Pg/Protocol/PgClientFlow.cs | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index 4a81211..233cd65 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -34,10 +34,10 @@ public enum RowBuffering : byte PgError? _errorMessage; Action? _completionAction; object? _completionActionState; - CommandFlow _flow = null!; + PgClientFlow _flow = null!; // The requested row description is what was returned for this exact command (i.e. commands that requested a describe). - internal void Initialize(CommandFlow flow, int index, CommandDescriptor descriptor, + internal void Initialize(PgClientFlow flow, int index, CommandDescriptor descriptor, RowDescription? requestedRowDescription, bool requestedExecution, bool simpleProtocol, PgError? error = null) { if (!ReferenceEquals(_flow, flow)) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index c680196..ea980b3 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -1171,7 +1171,7 @@ void FaultCaller(Exception exception) CompleteEnumerationWithException(exception); } - internal void Fail(Exception exception) => FaultCaller(exception); + internal override void Fail(Exception exception) => FaultCaller(exception); protected override void OnReleasing(Exception? exception) { diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 341c6b6..c510cfc 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -431,6 +431,10 @@ public void Reset() protected virtual void OnHeartbeat(TimeSpan interval) {} protected virtual void OnAbort(Exception exception) {} + /// Terminates the flow from a result callback. Result-producing flows route the fault to their + /// consumer and framework completion. Other flows never hand out results. + internal virtual void Fail(Exception exception) + => throw new InvalidOperationException("This flow does not produce command results.", exception); /// Graceful-shutdown observation point. Fires while StoppingToken is set but before the /// AbortToken escalation. Flow types whose body can park on a non-IO rendezvous (CommandFlow's /// GateTask) override this to wake it so the body short-circuits instead of waiting for From be006e6ac81b1f434aba59d1828ada2885a7a4ce Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 28 Aug 2026 16:44:49 +0200 Subject: [PATCH 015/136] Write prepared execution as one reserved span --- .../Pg/PgEncoderPreparedExecutionTests.cs | 111 ++++++++++++++++++ Slon/Pg/Protocol/Flows/CommandExtensions.cs | 17 +++ Slon/Pg/Protocol/PgEncoder.cs | 84 +++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs diff --git a/Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs b/Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs new file mode 100644 index 0000000..25c4a16 --- /dev/null +++ b/Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs @@ -0,0 +1,111 @@ +using System.Buffers.Binary; +using System.Text; +using Slon.Pg.Protocol; +using Slon.Text; + +namespace Slon.Tests.Pg; + +// The fused prepared-execution write must produce exactly the bytes the message-per-message writers +// produce, for every Describe, Execute, and Sync combination, and must leave the writer's +// per-message framing validation intact. +[TestClass] +public class PgEncoderPreparedExecutionTests +{ + static readonly Encoding Encoding = Encoding.UTF8; + + static (ProtocolDataWriter Writer, BufferOutputWriter Sink) NewWriter() + { + var sink = new BufferOutputWriter(); + var writer = new ProtocolDataWriter(sink, Encoding, static _ => { }, default, null!); + return (writer, sink); + } + + static byte[] Expected(string commandName, bool describe, bool execute, int syncCount) + { + var bytes = new List(); + var name = Encoding.GetBytes(commandName); + + // Bind: unnamed portal, statement name, no parameter format codes, no parameters, one result + // format code, binary. + Message(bytes, (byte)'B', [0, .. name, 0, 0, 0, 0, 0, 0, 1, 0, 1]); + if (describe) + Message(bytes, (byte)'D', [(byte)'P', 0]); + if (execute) + Message(bytes, (byte)'E', [0, 0, 0, 0, 0]); + for (var i = 0; i < syncCount; i++) + Message(bytes, (byte)'S', []); + return bytes.ToArray(); + + static void Message(List bytes, byte type, ReadOnlySpan body) + { + bytes.Add(type); + Span length = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32BigEndian(length, (uint)(sizeof(uint) + body.Length)); + bytes.AddRange(length); + bytes.AddRange(body); + } + } + + [TestMethod] + [DataRow(false, false, 0)] + [DataRow(false, false, 1)] + [DataRow(false, false, 2)] + [DataRow(false, true, 0)] + [DataRow(false, true, 1)] + [DataRow(false, true, 2)] + [DataRow(true, false, 0)] + [DataRow(true, false, 1)] + [DataRow(true, false, 2)] + [DataRow(true, true, 0)] + [DataRow(true, true, 1)] + [DataRow(true, true, 2)] + public void WritesExactWireBytes(bool describe, bool execute, int syncCount) + { + var (writer, sink) = NewWriter(); + + PgEncoder.WritePreparedExecutionCore(writer, Encoding, new EncodedCString("prepared_probe"), + describe, execute, syncCount); + writer.Flush(); + + CollectionAssert.AreEqual(Expected("prepared_probe", describe, execute, syncCount), sink.ToArray()); + } + + [TestMethod] + public void UnnamedStatement_WritesEmptyName() + { + var (writer, sink) = NewWriter(); + + PgEncoder.WritePreparedExecutionCore(writer, Encoding, default, describe: false, execute: true, syncCount: 1); + writer.Flush(); + + CollectionAssert.AreEqual(Expected("", describe: false, execute: true, syncCount: 1), sink.ToArray()); + } + + [TestMethod] + public void EveryMessageIsFramedForTheDeclaredLengthCheck() + { + var (writer, sink) = NewWriter(); + + PgEncoder.WritePreparedExecutionCore(writer, Encoding, new EncodedCString("prepared_probe"), + describe: true, execute: true, syncCount: 2); + // Arming the next message validates that the previous one was written to its declared length. + writer.StartMessage(totalLength: 5); + writer.WriteRaw(new byte[5]); + writer.Flush(); + + Assert.AreEqual(Expected("prepared_probe", describe: true, execute: true, syncCount: 2).Length + 5, + sink.ToArray().Length); + } + + [TestMethod] + [DataRow(-1)] + [DataRow(3)] + public void SyncCount_OutsideZeroToTwo_Throws(int syncCount) + { + var (writer, _) = NewWriter(); + + Assert.ThrowsExactly(() => + PgEncoder.WritePreparedExecutionCore(writer, Encoding, new EncodedCString("prepared_probe"), + describe: false, execute: true, syncCount)); + } +} diff --git a/Slon/Pg/Protocol/Flows/CommandExtensions.cs b/Slon/Pg/Protocol/Flows/CommandExtensions.cs index 4c09b5b..f989dcc 100644 --- a/Slon/Pg/Protocol/Flows/CommandExtensions.cs +++ b/Slon/Pg/Protocol/Flows/CommandExtensions.cs @@ -12,6 +12,23 @@ public static class CommandExtensions public static bool IsSimple(this in Command command) => command.PreferSimple && command.WithSync && !command.DescribeOnly && !command.Descriptor.IsPrepared && command.Descriptor.ParameterTypes.Count is 0; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool CanWritePreparedExecution(in Command command, in CommandDescriptor descriptor) + => descriptor.IsPrepared && command.Parameters.Count is 0 + && descriptor.ParameterTypes.Count is 0 && command.ResultFormats.Length is 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static ValueTask WritePreparedExecutionAsync( + in Command command, in CommandDescriptor descriptor, PgEncoder encoder, bool appendSync, + CancellationToken cancellationToken) + { + encoder.WritePreparedExecution(descriptor.CommandName, + describe: command.DescribeOnly || descriptor.PreparedRowDescription is null, + execute: !command.DescribeOnly, + syncCount: (command.WithSync ? 1 : 0) + (appendSync ? 1 : 0)); + return encoder.FlushAsync(cancellationToken); + } + // Sync/async pair at the command-list (full composition) level. No *Auto wrapper here. // Callers picking sync vs async make that choice once at this level rather than threading // a mode flag through every encoder helper underneath. Keeping the list loop in this state diff --git a/Slon/Pg/Protocol/PgEncoder.cs b/Slon/Pg/Protocol/PgEncoder.cs index 31c4d30..165e87c 100644 --- a/Slon/Pg/Protocol/PgEncoder.cs +++ b/Slon/Pg/Protocol/PgEncoder.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Collections.Immutable; using System.Runtime.CompilerServices; using System.Text; @@ -233,6 +234,89 @@ public void WriteBind(EncodedCString commandName) _writer.WriteUShort(1); // all binary } + // Executes a prepared statement without parameters and with default result formats: Bind on the + // unnamed portal, an optional portal Describe, an optional Execute, then syncCount Sync messages, + // all written through one reserved span. A flow appends at most one Sync of its own beside the + // command's, so syncCount is bounded to two. + internal void WritePreparedExecution(EncodedCString commandName, bool describe, bool execute, int syncCount) + { + WritePreparedExecutionCore(_writer, ClientEncoding, commandName, describe, execute, syncCount); + _executionControl.OnMessageWrite(FrontendType.Bind); + if (describe) + _executionControl.OnMessageWrite(FrontendType.Describe); + if (execute) + _executionControl.OnMessageWrite(FrontendType.Execute); + for (var i = 0; i < syncCount; i++) + _executionControl.OnMessageWrite(FrontendType.Sync); + } + + // Each message is still armed and advanced on its own so the per-message declared-length check + // holds. The reserved span stays valid across the advances because the buffering writer only + // reallocates on a reservation it cannot satisfy. + internal static void WritePreparedExecutionCore(ProtocolDataWriter writer, Encoding encoding, + EncodedCString commandName, bool describe, bool execute, int syncCount) + { + ArgumentOutOfRangeException.ThrowIfNegative(syncCount); + ArgumentOutOfRangeException.ThrowIfGreaterThan(syncCount, 2); + + const int header = sizeof(byte) + sizeof(uint); + const int describeBody = sizeof(byte) + 1; // 'P' and the unnamed portal + const int executeBody = sizeof(byte) + sizeof(int); // unnamed portal, all rows + var commandNameBytes = commandName.AsNullTerminatedSpan(encoding); + var bindBody = checked(1 + commandNameBytes.Length + 4 * sizeof(ushort)); + var total = checked(header + bindBody + + (describe ? header + describeBody : 0) + + (execute ? header + executeBody : 0) + + syncCount * header); + var span = writer.GetSpan(total); + + writer.StartMessage(header + bindBody); + WriteHeader(span, FrontendType.Bind, bindBody); + span[header] = 0; // unnamed portal + commandNameBytes.CopyTo(span.Slice(header + 1)); + var formats = span.Slice(header + 1 + commandNameBytes.Length); + BinaryPrimitives.WriteUInt16BigEndian(formats, 0); // parameter format codes + BinaryPrimitives.WriteUInt16BigEndian(formats.Slice(2), 0); // parameters + BinaryPrimitives.WriteUInt16BigEndian(formats.Slice(4), 1); // result format codes + BinaryPrimitives.WriteUInt16BigEndian(formats.Slice(6), 1); // all binary + writer.Advance(header + bindBody); + span = span.Slice(header + bindBody); + + if (describe) + { + writer.StartMessage(header + describeBody); + WriteHeader(span, FrontendType.Describe, describeBody); + span[header] = (byte)'P'; + span[header + 1] = 0; + writer.Advance(header + describeBody); + span = span.Slice(header + describeBody); + } + + if (execute) + { + writer.StartMessage(header + executeBody); + WriteHeader(span, FrontendType.Execute, executeBody); + span[header] = 0; // unnamed portal + BinaryPrimitives.WriteUInt32BigEndian(span.Slice(header + 1), 0); // all rows + writer.Advance(header + executeBody); + span = span.Slice(header + executeBody); + } + + for (var i = 0; i < syncCount; i++) + { + writer.StartMessage(header); + WriteHeader(span, FrontendType.Sync, 0); + writer.Advance(header); + span = span.Slice(header); + } + + static void WriteHeader(Span span, FrontendType type, int bodyLength) + { + span[0] = type.ToByte(); + BinaryPrimitives.WriteUInt32BigEndian(span.Slice(1), checked((uint)(sizeof(uint) + bodyLength))); + } + } + public void WriteBind(EncodedCString commandName = default, EncodedCString portalName = default, ParameterSource parameters = default, ImmutableArray resultFormats = default) { From 6efbe94d0263274a895b352f7712572933195062 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 28 Aug 2026 16:56:33 +0200 Subject: [PATCH 016/136] Keep shared read state across compatible idle edges --- Slon/Pg/Protocol/PgClientFlow.cs | 4 ++++ Slon/Pg/Protocol/PgClientProtocol.cs | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index c510cfc..2fc4374 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -431,6 +431,10 @@ public void Reset() protected virtual void OnHeartbeat(TimeSpan interval) {} protected virtual void OnAbort(Exception exception) {} + /// True when the flow resets the protocol-static read objects before its pipeline task completes + /// and hands out no result past its terminal. The idle edge then keeps those objects for the next + /// flow instead of replacing them to protect a handle retained past completion. + internal virtual bool ResetsSharedReadStateBeforeRelease => false; /// Terminates the flow from a result callback. Result-producing flows route the fault to their /// consumer and framework completion. Other flows never hand out results. internal virtual void Fail(Exception exception) diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index 853c1ed..fe29124 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1846,8 +1846,9 @@ internal void OnReleasing(PgClientFlow flow) flow.GetExecutionControl(this).StallsPipeline); // Draghi clears the activated slot only at the exact idle edge and before CompleteItem. - // Release the shared read objects before the flow's terminal observer can reuse them. - if (idle) + // Release the shared read objects before the flow's terminal observer can reuse them, + // unless the flow already reset them and hands out nothing past its terminal. + if (idle && !flow.ResetsSharedReadStateBeforeRelease) _commandFlowReadState = new(); } From 7b2e8e75457eef6c93d9ce996258dc8df43203ee Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Mon, 31 Aug 2026 16:34:29 +0200 Subject: [PATCH 017/136] Allow protocol hosts to drive heartbeat ticks --- Slon.Tests/Pg/HeartbeatTests.cs | 28 ++++++++++++++++++++++++++++ Slon/Pg/Protocol/PgClientProtocol.cs | 14 +++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Slon.Tests/Pg/HeartbeatTests.cs b/Slon.Tests/Pg/HeartbeatTests.cs index 562e425..e614e01 100644 --- a/Slon.Tests/Pg/HeartbeatTests.cs +++ b/Slon.Tests/Pg/HeartbeatTests.cs @@ -10,6 +10,22 @@ namespace Slon.Tests.Pg; [TestClass] public class HeartbeatTests { + [ConnectionCreatingTestMethod] + [DataRow(PgClientProtocolHeartbeatMode.Automatic, 1)] + [DataRow(PgClientProtocolHeartbeatMode.External, 0)] + public async Task ProtocolHeartbeatModeControlsTimerOwnership( + PgClientProtocolHeartbeatMode mode, int expectedTimerCount) + { + var time = new CountingTimeProvider(); + await using var protocol = await PgTestPool.NewIsolatedAsync(options => + { + options.TimeProvider = time; + options.HeartbeatMode = mode; + }); + + Assert.AreEqual(expectedTimerCount, time.TimerCount); + } + [TestMethod] public async Task BackloggedFlow_ActivationTimeoutAdvancesBeforeDispatch() { @@ -94,4 +110,16 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, } readonly record struct Entry(LogLevel Level, Exception? Exception, string Message); + + sealed class CountingTimeProvider : FakeTimeProvider + { + internal int TimerCount { get; private set; } + + public override ITimer CreateTimer(TimerCallback callback, object? state, + TimeSpan dueTime, TimeSpan period) + { + TimerCount++; + return base.CreateTimer(callback, state, dueTime, period); + } + } } diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index fe29124..ffcd2f8 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -36,6 +36,13 @@ public enum FlowEnqueueOptions : byte AllowMigration = 4 } +[Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] +public enum PgClientProtocolHeartbeatMode : byte +{ + Automatic, + External +} + interface IProtocolStatic { ref readonly T Value { get; } @@ -79,6 +86,7 @@ public PgClientProtocolOptions(PgClientOptions options) public Encoding DefaultClientEncoding { get; set; } public TimeSpan FlowActivationTimeout { get; set; } public TimeSpan HeartbeatInterval { get; set; } = Heartbeat.DefaultInterval; + public PgClientProtocolHeartbeatMode HeartbeatMode { get; set; } public TimeSpan ReadTimeout { get; set; } = PgClientOptions.DefaultReadTimeout; public TimeSpan WriteTimeout { get; set; } = TimeSpan.FromSeconds(10); // Allocation-free grace before starting a backend CancelRequest. Heartbeat supplies the clock. @@ -312,7 +320,8 @@ void Initialize(TransportConnection connection, Hosting hosting) _admissionAvailable = hosting.AdmissionAvailable; _loadObserver = hosting.LoadObserver; - if (!hosting.DrivesHeartbeat) + if (!hosting.DrivesHeartbeat && + _options.HeartbeatMode is PgClientProtocolHeartbeatMode.Automatic) { _heartbeat = new(_options.HeartbeatInterval, _options.TimeProvider, _logger); _heartbeat.Register(period => Heartbeat(period)); @@ -1124,6 +1133,9 @@ internal ValueTask Heartbeat(TimeSpan period) return new(); } + public ValueTask HeartbeatAsync(TimeSpan elapsed) + => Heartbeat(elapsed); + void PropagateFlowHeartbeat(TimeSpan period) { var control = FlowControl; From ea33cc3867dcc8ec0fa9981f53cb4299bed68579 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 01:55:05 +0200 Subject: [PATCH 018/136] Add decoder-owned value row collection --- Slon/Pg/CommandResult.cs | 143 ++++++++++++++--- Slon/Pg/Protocol/BackendMessage.cs | 45 +++++- Slon/Pg/Protocol/BackendMessageContext.cs | 67 ++++++++ .../Flows/CommandFlow.MessageEnumerator.cs | 145 +++++++++++++++++- Slon/Pg/Protocol/PgDecoder.cs | 19 +++ Slon/Pg/Protocol/ProtocolReadPipe.cs | 8 + Slon/Pg/Row.cs | 61 ++++++-- 7 files changed, 447 insertions(+), 41 deletions(-) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index 233cd65..dcb9355 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -1,9 +1,11 @@ +using System.Buffers.Binary; using System.Collections; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; +using Slon.Pg.Types; namespace Slon.Pg; @@ -256,6 +258,89 @@ internal async ValueTask CompleteAsync() EnsureComplete(); } + internal readonly struct RowView + { + readonly byte[]? _array; + readonly ReadOnlyMemory _memory; + readonly int _offset; + readonly int _length; + + internal RowView(ReadOnlyMemory memory) + => (_array, _memory, _offset, _length) = (null, memory, 0, 0); + + internal RowView(byte[] array, int offset, int length) + => (_array, _memory, _offset, _length) = (array, default, offset, length); + + public T GetValue(int ordinal) + => BootstrapFieldDecoder.Read(GetFieldSpan(ordinal)); + + public int GetInt32(int ordinal) + { + if (ordinal == 0) + { + var row = _array is { } array + ? array.AsSpan(_offset, _length) + : _memory.Span; + const int valueOffset = sizeof(short) + sizeof(int); + if (row.Length >= valueOffset + sizeof(int) + && BinaryPrimitives.ReadInt32BigEndian(row[sizeof(short)..]) == sizeof(int)) + return BinaryPrimitives.ReadInt32BigEndian(row[valueOffset..]); + ThrowHelper.ThrowInvalidOperation("The first DataRow field is not a non-null Int32."); + } + return BinaryPrimitives.ReadInt32BigEndian(GetFieldSpan(ordinal)); + } + + ReadOnlySpan GetFieldSpan(int ordinal) + { + ArgumentOutOfRangeException.ThrowIfNegative(ordinal); + var fields = _array is { } array + ? array.AsSpan(_offset, _length) + : _memory.Span; + if (fields.Length >= sizeof(short)) + { + var remaining = fields[sizeof(short)..]; + for (var index = 0; ; index++) + { + if (remaining.Length < sizeof(int)) + ThrowHelper.ThrowInvalidOperation("The DataRow field length is truncated."); + var length = BinaryPrimitives.ReadInt32BigEndian(remaining); + remaining = remaining[sizeof(int)..]; + if (length < 0) + { + if (index == ordinal) + ThrowHelper.ThrowInvalidOperation("The requested field is null."); + continue; + } + if ((uint)length > (uint)remaining.Length) + ThrowHelper.ThrowInvalidOperation("The DataRow field is truncated."); + if (index == ordinal) + return remaining[..length]; + remaining = remaining[length..]; + } + } + + ThrowHelper.ThrowInvalidOperation("The DataRow field count is truncated."); + return default; + } + } + + internal async ValueTask CollectRowsAsync( + TState state, Action collector, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(collector); + cancellationToken.ThrowIfCancellationRequested(); + if (_firstRowEnumerated) + ThrowHelper.ThrowInvalidOperation( + "Rows have already been consumed from this command result."); + _firstRowEnumerated = true; + var terminal = await _messageEnumerator + .CollectRowsAsync(state, collector, cancellationToken) + .ConfigureAwait(false); + CompleteTerminal(terminal); + _messageEnumerator.ThrowCollectorException(); + } + void EnsureComplete() { if (_requestedExecution && !IsComplete) @@ -329,6 +414,7 @@ Row GetRow() } BackendMessage GetCurrentMessage() => _messageEnumerator.Current; + BackendMessage.Accessor GetCurrentMessageAccessor() => _messageEnumerator.CurrentAccessor; bool MoveNextMessage() => _messageEnumerator.MoveNext(); CommandFlow.MoveNextStatus TryMoveNextMessage() => _messageEnumerator.TryMoveNext(); ValueTask MoveNextMessageAsync() => _messageEnumerator.MoveNextAsync(); @@ -342,17 +428,17 @@ public struct RowEnumerator : IEnumerator, IAsyncEnumerator internal RowEnumerator(CommandResult instance, RowBuffering buffering) => (_instance, _buffering) = (instance, buffering); - BackendMessage PrepareRow(BackendMessage message) + BackendMessage.Accessor PrepareRow(BackendMessage.Accessor message) { if (_buffering is RowBuffering.Buffered && !message.Buffered) { message.BufferBody(); - message = _instance!.GetCurrentMessage(); + message = _instance!.GetCurrentMessageAccessor(); } return message; } - bool PublishRow(in BackendMessage message) + bool PublishRow(in BackendMessage.Accessor message) { (_row ??= _instance!.GetRow()).InitializeRow(message); return true; @@ -377,8 +463,8 @@ public bool MoveNext() // https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY // "Therefore, an Execute phase is always terminated by the appearance of exactly one of these messages: // CommandComplete, EmptyQueryResponse (if the portal was created from an empty query string), ErrorResponse, or PortalSuspended" - var current = instance.GetCurrentMessage(); - if (current.Header.Type is PgTypes.BackendType.DataRow) + var current = instance.GetCurrentMessageAccessor(); + if (current.Type is PgTypes.BackendType.DataRow) return PublishRow(PrepareRow(current)); return HandleUncommon(current); @@ -408,21 +494,14 @@ public ValueTask MoveNextAsync() return MoveNextAfterRevokeAsync(leasedRow); var status = instance.TryMoveNextMessage(); - if (status is CommandFlow.MoveNextStatus.RequiresInput) - return MoveNextAsyncCore(instance.MoveNextMessageAsync()); - - if (status is CommandFlow.MoveNextStatus.EndOfSequence) - { - if (instance._requestedExecution && instance._commandCompleteMessage is null && instance._errorMessage is null) - ThrowHelper.ThrowInvalidOperation("Underlying message enumerator completed before CommandComplete was returned."); - return new(false); - } + if (status is not CommandFlow.MoveNextStatus.Moved) + return HandleNonMoved(instance, status); // https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY // "Therefore, an Execute phase is always terminated by the appearance of exactly one of these messages: // CommandComplete, EmptyQueryResponse (if the portal was created from an empty query string), ErrorResponse, or PortalSuspended" - var current = instance.GetCurrentMessage(); - if (current.Header.Type is PgTypes.BackendType.DataRow) + var current = instance.GetCurrentMessageAccessor(); + if (current.Type is PgTypes.BackendType.DataRow) { if (_buffering is RowBuffering.Buffered && !current.Buffered) return BufferCurrentRow(in current); @@ -433,13 +512,24 @@ public ValueTask MoveNextAsync() } [MethodImpl(MethodImplOptions.NoInlining)] - ValueTask BufferCurrentRow(in BackendMessage current) + ValueTask HandleNonMoved(CommandResult instance, CommandFlow.MoveNextStatus status) + { + if (status is CommandFlow.MoveNextStatus.RequiresInput) + return MoveNextAsyncCore(instance.MoveNextMessageAsync()); + + if (instance._requestedExecution && instance._commandCompleteMessage is null && instance._errorMessage is null) + ThrowHelper.ThrowInvalidOperation("Underlying message enumerator completed before CommandComplete was returned."); + return new(false); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + ValueTask BufferCurrentRow(in BackendMessage.Accessor current) { var instance = _instance!; var bufferTask = current.BufferBodyAsync(default); if (!bufferTask.IsCompletedSuccessfully) return BufferRowAsync(bufferTask, _row ??= instance.GetRow()); - return new(PublishRow(instance.GetCurrentMessage())); + return new(PublishRow(instance.GetCurrentMessageAccessor())); } [MethodImpl(MethodImplOptions.NoInlining)] @@ -450,16 +540,16 @@ async ValueTask MoveNextAfterRevokeAsync(Row row) } [MethodImpl(MethodImplOptions.NoInlining)] - bool HandleUncommon(in BackendMessage current) + bool HandleUncommon(in BackendMessage.Accessor current) { var instance = _instance!; - var type = current.Header.Type; + var type = current.Type; switch (type) { case PgTypes.BackendType.EmptyQueryResponse: case PgTypes.BackendType.CommandComplete: case PgTypes.BackendType.ErrorResponse: - instance.CompleteCommand(current); + instance.CompleteCommand(current.Message); return false; case PgTypes.BackendType.PortalSuspended when !instance._simpleProtocol: default: @@ -483,12 +573,15 @@ async ValueTask MoveNextAsyncCore(ValueTask task) // https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-EXT-QUERY // "Therefore, an Execute phase is always terminated by the appearance of exactly one of these messages: // CommandComplete, EmptyQueryResponse (if the portal was created from an empty query string), ErrorResponse, or PortalSuspended" - var current = instance.GetCurrentMessage(); - if (current.Header.Type is PgTypes.BackendType.DataRow) + var current = instance.GetCurrentMessageAccessor(); + if (current.Type is PgTypes.BackendType.DataRow) { if (_buffering is RowBuffering.Buffered && !current.Buffered) + { await current.BufferBodyAsync(default).ConfigureAwait(false); - return PublishRow(instance.GetCurrentMessage()); + current = instance.GetCurrentMessageAccessor(); + } + return PublishRow(current); } return HandleUncommon(current); @@ -499,7 +592,7 @@ async ValueTask MoveNextAsyncCore(ValueTask task) async ValueTask BufferRowAsync(ValueTask task, Row row) { await task.ConfigureAwait(false); - row.InitializeRow(_instance!.GetCurrentMessage()); + row.InitializeRow(_instance!.GetCurrentMessageAccessor()); return true; } diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index 4575f05..7c3ab67 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -215,6 +215,30 @@ internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory mem return false; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGetBufferedFirstArray(int offset, + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, + out int start, out int length) + { + Debug.Assert(Buffered); + offset += BackendHeader.ByteCount; + var firstLength = IsIndependent + ? _endIndexOrBufferedLength - _startIndex + : _endIndexOrBufferedLength; + if (_firstObject is byte[] firstArray && (uint)offset <= (uint)firstLength) + { + array = firstArray; + start = _startIndex + offset; + length = firstLength - offset; + return true; + } + + array = null; + start = 0; + length = 0; + return false; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] ReadOnlyMemory GetFirstMemory() { @@ -263,7 +287,7 @@ static void Throw(BackendType actual, ReadOnlySpan expected) $"Unexpected backend message: {actual}, expected: {string.Join(" or ", expected.ToArray())}."); } - public Accessor GetAccessor() => new(Context, Token); + public Accessor GetAccessor() => new(Context, Token, Type, Buffered); internal BackendMessageBodyReader OpenBodyReader() => new(Context, Token, GetSequence(), Buffered); @@ -281,15 +305,30 @@ public readonly struct Accessor { readonly BackendMessageContext _context; readonly short _token; + readonly PgTypes.BackendType _type; + readonly bool _buffered; - internal Accessor(BackendMessageContext context, short token) + internal Accessor(BackendMessageContext context, short token, + PgTypes.BackendType type, bool buffered) { _context = context; _token = token; + _type = type; + _buffered = buffered; } public BackendMessage Message => _context.GetCurrent(_token); + internal PgTypes.BackendType Type => _type; + internal bool Buffered => _buffered; + internal BackendMessageBodyReader OpenBodyReader() + => _context.OpenCurrentBodyReader(_token); + internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory memory) + => _context.TryGetCurrentBufferedFirstMemory(_token, offset, out memory); + internal void BufferBody() => _context.BufferCurrentMessage(_token); + internal ValueTask BufferBodyAsync(CancellationToken cancellationToken) + => _context.BufferCurrentMessageAsync(_token, cancellationToken); + // The JIT should have a phase for picking granular writes (and write barriers) over full struct assignments. // This translation is entirely mechanical (even though these implementations need to deviate for external types). internal static void WriteGranularly(ref Accessor destination, in Accessor value, bool destinationIsZero = false) @@ -298,6 +337,8 @@ internal static void WriteGranularly(ref Accessor destination, in Accessor value Unsafe.AsRef(in destination._context) = value._context!; Unsafe.AsRef(in destination._token) = value._token; + Unsafe.AsRef(in destination._type) = value._type; + Unsafe.AsRef(in destination._buffered) = value._buffered; } } diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 0697901..43af08a 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -81,6 +81,17 @@ public BackendMessage Current } } + public BackendMessage.Accessor CurrentAccessor + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (_publicationState is not PublicationState.Current) + ThrowHelper.ThrowInvalidOperation("The decoder has no current backend message."); + return new(this, _version, _current.Header.Type, _current.Buffered); + } + } + public bool CurrentIsError { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -91,6 +102,47 @@ public bool CurrentIsError } } + public PgTypes.BackendType CurrentType + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + Debug.Assert(_publicationState is PublicationState.Current); + return _current.Header.Type; + } + } + + public bool CurrentBuffered + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + Debug.Assert(_publicationState is PublicationState.Current); + return _current.Buffered; + } + } + + public ReadOnlyMemory CurrentBufferedBody + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + Debug.Assert(_publicationState is PublicationState.Current); + if (!_current.TryGetBufferedFirstMemory(0, out var body)) + ThrowHelper.ThrowInvalidOperation("The current backend message is not buffered contiguously."); + return body; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryGetCurrentBufferedArray( + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, + out int start, out int length) + { + Debug.Assert(_publicationState is PublicationState.Current); + return _current.TryGetBufferedFirstArray(0, out array, out start, out length); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGetCurrent(out BackendMessage current) { @@ -105,6 +157,21 @@ public BackendMessage GetCurrent(short token) return _current; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal BackendMessageBodyReader OpenCurrentBodyReader(short token) + { + Validate(token); + return _current.OpenBodyReader(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGetCurrentBufferedFirstMemory(short token, int offset, + out ReadOnlyMemory memory) + { + Validate(token); + return _current.TryGetBufferedFirstMemory(offset, out memory); + } + internal void SetCurrentFallbackBuffer( in ReadOnlySequence buffer, bool required) { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 7b799c9..a6632f7 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -46,7 +46,13 @@ internal readonly struct ResultMessageEnumerator() : IEnumerator public bool MoveNext() => _messageEnumerator.MoveNext(); public ValueTask MoveNextAsync() => _messageEnumerator.MoveNextAsync(); public BackendMessage Current => _messageEnumerator.Current; + internal BackendMessage.Accessor CurrentAccessor => _messageEnumerator.CurrentAccessor; internal MoveNextStatus TryMoveNext() => _messageEnumerator.TryMoveNext(); + internal ValueTask CollectRowsAsync( + TState state, Action collector, + CancellationToken cancellationToken) + => _messageEnumerator.CollectRowsAsync(state, collector, cancellationToken); + internal void ThrowCollectorException() => _messageEnumerator.ThrowCollectorException(); public void Dispose() => _messageEnumerator.Dispose(); public ValueTask DisposeAsync() => _messageEnumerator.DisposeAsync(); @@ -78,6 +84,7 @@ sealed class MessageEnumerator : IEnumerator, IAsyncEnumerator Core() } } + enum CollectRowsStatus : byte + { + RequiresInput, + RequiresBuffer, + Complete + } + + public ValueTask CollectRowsAsync( + TState state, Action collector, + CancellationToken cancellationToken) + { + try + { + var status = CollectAvailableRows( + state, collector, currentReady: false, out var pending, out var terminal); + return status is CollectRowsStatus.Complete + ? new(terminal) + : Core(this, state, collector, status, pending, cancellationToken); + } + catch (Exception ex) + { + _exceptionDispatchInfo = ExceptionDispatchInfo.Capture(ex); + return ValueTask.FromException(ex); + } + + static async ValueTask Core( + MessageEnumerator enumerator, + TState state, Action collector, + CollectRowsStatus status, BackendMessage.Accessor pending, + CancellationToken cancellationToken) + { + try + { + while (true) + { + if (status is CollectRowsStatus.RequiresInput) + { + cancellationToken.ThrowIfCancellationRequested(); + _ = await enumerator._decoder.GetNextAsync().ConfigureAwait(false); + } + else + { + await pending.BufferBodyAsync(cancellationToken).ConfigureAwait(false); + } + + status = enumerator.CollectAvailableRows( + state, collector, currentReady: true, out pending, out var terminal); + if (status is CollectRowsStatus.Complete) + return terminal; + } + } + catch (Exception ex) + { + enumerator._exceptionDispatchInfo = ExceptionDispatchInfo.Capture(ex); + throw; + } + } + } + + CollectRowsStatus CollectAvailableRows( + TState state, Action collector, + bool currentReady, + out BackendMessage.Accessor pending, + out BackendMessage terminal) + { + while (true) + { + if (!currentReady) + { + if (_first) + { + _first = false; + currentReady = true; + } + else + { + _exceptionDispatchInfo?.Throw(); + if (_done) + ThrowHelper.ThrowInvalidOperation( + "Underlying message enumerator completed before a terminal message was returned."); + if (!_decoder.TryMoveNext()) + { + pending = default; + terminal = default; + return CollectRowsStatus.RequiresInput; + } + currentReady = true; + } + } + + DebugEnsureExpected(_decoder.Current); + if (_decoder.CurrentType is not PgTypes.BackendType.DataRow) + { + _done = true; + pending = default; + terminal = _decoder.Current; + return CollectRowsStatus.Complete; + } + if (!_decoder.CurrentBuffered) + { + pending = _decoder.CurrentAccessor; + terminal = default; + return CollectRowsStatus.RequiresBuffer; + } + + if (_collectorException is null) + { + try + { + var row = _decoder.TryGetCurrentBufferedArray( + out var array, out var offset, out var length) + ? new CommandResult.RowView(array, offset, length) + : new CommandResult.RowView(_decoder.CurrentBufferedBody); + collector(state, row); + } + catch (Exception ex) + { + _collectorException = ExceptionDispatchInfo.Capture(ex); + } + } + currentReady = false; + } + } + + public void ThrowCollectorException() + { + var exception = _collectorException; + _collectorException = null; + exception?.Throw(); + } + public MoveNextStatus TryMoveNext() { if (_first) @@ -180,13 +318,14 @@ public MoveNextStatus TryMoveNext() return MoveNextStatus.RequiresInput; } - public BackendMessage Current { [MethodImpl(MethodImplOptions.AggressiveInlining)] get => _decoder.Current; } + internal BackendMessage.Accessor CurrentAccessor => _decoder.CurrentAccessor; + public void Dispose() { _exceptionDispatchInfo?.Throw(); @@ -294,6 +433,8 @@ public void Initialize(in Command command, PgDecoder decoder) _decoder = decoder; _exceptionDispatchInfo = null; + if (_collectorException is not null) + _collectorException = null; _disposed = false; _completeError = null; @@ -310,6 +451,8 @@ public void Reset() _withSync = false; _decoder = null!; _exceptionDispatchInfo = null; + if (_collectorException is not null) + _collectorException = null; _completeError = null; _disposed = true; _first = false; diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 2d5cf07..f32b018 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -725,6 +725,25 @@ public BackendMessage Current get => _pipe.Current; } + internal BackendMessage.Accessor CurrentAccessor + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => _pipe.CurrentAccessor; + } + + internal ReadOnlyMemory CurrentBufferedBody + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => _pipe.CurrentBufferedBody; + } + + internal PgTypes.BackendType CurrentType => _pipe.CurrentType; + internal bool CurrentBuffered => _pipe.CurrentBuffered; + internal bool TryGetCurrentBufferedArray( + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, + out int start, out int length) + => _pipe.TryGetCurrentBufferedArray(out array, out start, out length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGetCurrent(out BackendMessage message) { diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index dcb27fe..1712a87 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -27,7 +27,15 @@ enum PendingRead : byte { None, Messages, Slide, Extend } public PipeReader PipeReader => reader; public BackendMessage Current => _messageContext.Current; + public BackendMessage.Accessor CurrentAccessor => _messageContext.CurrentAccessor; public bool CurrentIsError => _messageContext.CurrentIsError; + public PgTypes.BackendType CurrentType => _messageContext.CurrentType; + public bool CurrentBuffered => _messageContext.CurrentBuffered; + public ReadOnlyMemory CurrentBufferedBody => _messageContext.CurrentBufferedBody; + public bool TryGetCurrentBufferedArray( + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, + out int start, out int length) + => _messageContext.TryGetCurrentBufferedArray(out array, out start, out length); public bool TryGetCurrent(out BackendMessage message) => _messageContext.TryGetCurrent(out message); diff --git a/Slon/Pg/Row.cs b/Slon/Pg/Row.cs index 8dd1536..5f3f88b 100644 --- a/Slon/Pg/Row.cs +++ b/Slon/Pg/Row.cs @@ -2,6 +2,7 @@ using System.Buffers.Binary; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; using Slon.Buffers; using Slon.Pg.Protocol; @@ -19,7 +20,10 @@ internal Row() { } RowDescription _rowDescription = null!; BackendMessageBodyReader? _bodyReader; IColumnLease? _columnLease; + byte[]? _bufferedArray; ReadOnlyMemory _bufferedBody; + int _bufferedOffset; + int _bufferedLength; int _leasedOrdinal; int _lastBufferedOrdinal = -1; int _lastBufferedOffset; @@ -29,6 +33,10 @@ internal Row() { } int _columnOffset; BackendMessage Message => _messageAccessor.Message; + ReadOnlyMemory BufferedBody + => _bufferedArray is { } array + ? array.AsMemory(_bufferedOffset, _bufferedLength) + : _bufferedBody; [MethodImpl(MethodImplOptions.AggressiveInlining)] SequenceReader GetColumnReader(int ordinal, out int columnIndex, out int columnOffset) @@ -445,18 +453,19 @@ bool TryGetFieldMemory(int ordinal, out ReadOnlyMemory field) { if (ordinal == _lastBufferedOrdinal) { - field = _bufferedBody.Slice(_lastBufferedOffset, _lastBufferedLength); + field = BufferedBody.Slice(_lastBufferedOffset, _lastBufferedLength); return true; } var columnIndex = _column <= ordinal ? _column : 0; var columnOffset = _column <= ordinal ? _columnOffset : sizeof(short); - if ((uint)columnOffset > (uint)_bufferedBody.Length) + var bufferedBody = BufferedBody; + if ((uint)columnOffset > (uint)bufferedBody.Length) { field = default; return false; } - var remainingMemory = _bufferedBody.Slice(columnOffset); + var remainingMemory = bufferedBody.Slice(columnOffset); var remaining = remainingMemory.Span; while (columnIndex++ < ordinal) @@ -596,15 +605,21 @@ internal void Initialize(RowDescription rowDescription) _rowDescription = rowDescription; } - internal void InitializeRow(in BackendMessage row) + internal void InitializeRow(in BackendMessage.Accessor row) { if (_columnLease is not null) throw new InvalidOperationException("The previous column lease must be revoked before advancing the row."); - _bodyReader = row.Buffered ? null : row.OpenBodyReader(); + if (row.Buffered) + { + if (_bodyReader is not null) + _bodyReader = null; + } + else + _bodyReader = row.OpenBodyReader(); _column = 0; _columnOffset = sizeof(short); _lastBufferedOrdinal = -1; - BackendMessage.Accessor.WriteGranularly(ref _messageAccessor, row.GetAccessor()); + BackendMessage.Accessor.WriteGranularly(ref _messageAccessor, row); CaptureBufferedBody(row); } @@ -618,17 +633,37 @@ void EnsureBuffered() } void CaptureBufferedBody() - { - var message = Message; - CaptureBufferedBody(message); - } + => CaptureBufferedBody(_messageAccessor); - void CaptureBufferedBody(in BackendMessage message) + void CaptureBufferedBody(in BackendMessage.Accessor message) { if (_bodyReader is null && message.TryGetBufferedFirstMemory(0, out var body)) - _bufferedBody = body; + { + if (MemoryMarshal.TryGetArray(body, out var segment)) + { + if (!ReferenceEquals(_bufferedArray, segment.Array)) + _bufferedArray = segment.Array; + _bufferedOffset = segment.Offset; + _bufferedLength = segment.Count; + if (!_bufferedBody.IsEmpty) + _bufferedBody = default; + } + else + { + if (_bufferedArray is not null) + _bufferedArray = null; + _bufferedBody = body; + } + } else - _bufferedBody = default; + { + if (_bufferedArray is not null) + _bufferedArray = null; + if (!_bufferedBody.IsEmpty) + _bufferedBody = default; + _bufferedOffset = 0; + _bufferedLength = 0; + } } // Returns false when the seek was exhausted, true if positioned correctly, and throws if the seek is invalid. From 3f91857f6a39b62118619578abbdc2c0694a5ba5 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 13:20:22 +0200 Subject: [PATCH 019/136] Keep collected row values memory-backed --- Slon/Pg/CommandResult.cs | 16 ++------- Slon/Pg/Protocol/BackendMessage.cs | 33 +++++-------------- Slon/Pg/Protocol/BackendMessageContext.cs | 18 ++++------ .../Flows/CommandFlow.MessageEnumerator.cs | 6 +--- Slon/Pg/Protocol/PgDecoder.cs | 4 --- Slon/Pg/Protocol/ProtocolReadPipe.cs | 4 --- 6 files changed, 19 insertions(+), 62 deletions(-) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index dcb9355..82c2ed7 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -260,16 +260,10 @@ internal async ValueTask CompleteAsync() internal readonly struct RowView { - readonly byte[]? _array; readonly ReadOnlyMemory _memory; - readonly int _offset; - readonly int _length; internal RowView(ReadOnlyMemory memory) - => (_array, _memory, _offset, _length) = (null, memory, 0, 0); - - internal RowView(byte[] array, int offset, int length) - => (_array, _memory, _offset, _length) = (array, default, offset, length); + => _memory = memory; public T GetValue(int ordinal) => BootstrapFieldDecoder.Read(GetFieldSpan(ordinal)); @@ -278,9 +272,7 @@ public int GetInt32(int ordinal) { if (ordinal == 0) { - var row = _array is { } array - ? array.AsSpan(_offset, _length) - : _memory.Span; + var row = _memory.Span; const int valueOffset = sizeof(short) + sizeof(int); if (row.Length >= valueOffset + sizeof(int) && BinaryPrimitives.ReadInt32BigEndian(row[sizeof(short)..]) == sizeof(int)) @@ -293,9 +285,7 @@ public int GetInt32(int ordinal) ReadOnlySpan GetFieldSpan(int ordinal) { ArgumentOutOfRangeException.ThrowIfNegative(ordinal); - var fields = _array is { } array - ? array.AsSpan(_offset, _length) - : _memory.Span; + var fields = _memory.Span; if (fields.Length >= sizeof(short)) { var remaining = fields[sizeof(short)..]; diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index 7c3ab67..cdb1f8a 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -204,38 +204,23 @@ internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory mem { Debug.Assert(Buffered); offset += BackendHeader.ByteCount; - var firstMemory = GetFirstMemory(); - if ((uint)offset <= (uint)firstMemory.Length) + var firstLength = IsIndependent + ? _endIndexOrBufferedLength - _startIndex + : _endIndexOrBufferedLength; + if (_firstObject is byte[] array && (uint)offset <= (uint)firstLength) { - memory = firstMemory.Slice(offset); + memory = array.AsMemory(_startIndex + offset, firstLength - offset); return true; } - memory = default; - return false; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool TryGetBufferedFirstArray(int offset, - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, - out int start, out int length) - { - Debug.Assert(Buffered); - offset += BackendHeader.ByteCount; - var firstLength = IsIndependent - ? _endIndexOrBufferedLength - _startIndex - : _endIndexOrBufferedLength; - if (_firstObject is byte[] firstArray && (uint)offset <= (uint)firstLength) + var firstMemory = GetFirstMemory(); + if ((uint)offset <= (uint)firstMemory.Length) { - array = firstArray; - start = _startIndex + offset; - length = firstLength - offset; + memory = firstMemory.Slice(offset); return true; } - array = null; - start = 0; - length = 0; + memory = default; return false; } diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 43af08a..3e1bfad 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -128,19 +128,13 @@ public ReadOnlyMemory CurrentBufferedBody get { Debug.Assert(_publicationState is PublicationState.Current); - if (!_current.TryGetBufferedFirstMemory(0, out var body)) - ThrowHelper.ThrowInvalidOperation("The current backend message is not buffered contiguously."); - return body; - } - } + if (_current.TryGetBufferedFirstMemory(0, out var body) + && body.Length == _current.Header.BodyLength) + return body; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGetCurrentBufferedArray( - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, - out int start, out int length) - { - Debug.Assert(_publicationState is PublicationState.Current); - return _current.TryGetBufferedFirstArray(0, out array, out start, out length); + var sequence = _current.GetSequence(); + return _current.GetContiguousMemory(sequence); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index a6632f7..6becaba 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -273,11 +273,7 @@ CollectRowsStatus CollectAvailableRows( { try { - var row = _decoder.TryGetCurrentBufferedArray( - out var array, out var offset, out var length) - ? new CommandResult.RowView(array, offset, length) - : new CommandResult.RowView(_decoder.CurrentBufferedBody); - collector(state, row); + collector(state, new CommandResult.RowView(_decoder.CurrentBufferedBody)); } catch (Exception ex) { diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index f32b018..43db1a5 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -739,10 +739,6 @@ internal ReadOnlyMemory CurrentBufferedBody internal PgTypes.BackendType CurrentType => _pipe.CurrentType; internal bool CurrentBuffered => _pipe.CurrentBuffered; - internal bool TryGetCurrentBufferedArray( - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, - out int start, out int length) - => _pipe.TryGetCurrentBufferedArray(out array, out start, out length); [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGetCurrent(out BackendMessage message) diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index 1712a87..f6a381b 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -32,10 +32,6 @@ enum PendingRead : byte { None, Messages, Slide, Extend } public PgTypes.BackendType CurrentType => _messageContext.CurrentType; public bool CurrentBuffered => _messageContext.CurrentBuffered; public ReadOnlyMemory CurrentBufferedBody => _messageContext.CurrentBufferedBody; - public bool TryGetCurrentBufferedArray( - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out byte[]? array, - out int start, out int length) - => _messageContext.TryGetCurrentBufferedArray(out array, out start, out length); public bool TryGetCurrent(out BackendMessage message) => _messageContext.TryGetCurrent(out message); From 5c35d8ece6ce606dba7061a3bb8857be6d16a6a9 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 13:28:39 +0200 Subject: [PATCH 020/136] Keep general row memory projection off the array path --- Slon/Pg/Protocol/BackendMessage.cs | 18 ++++++++++++++++++ Slon/Pg/Protocol/BackendMessageContext.cs | 17 +++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index cdb1f8a..c5c7ac9 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -224,6 +224,24 @@ internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory mem return false; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGetBufferedArrayMemory(int offset, out ReadOnlyMemory memory) + { + Debug.Assert(Buffered); + offset += BackendHeader.ByteCount; + var firstLength = IsIndependent + ? _endIndexOrBufferedLength - _startIndex + : _endIndexOrBufferedLength; + if (_firstObject is byte[] array && (uint)offset <= (uint)firstLength) + { + memory = array.AsMemory(_startIndex + offset, firstLength - offset); + return true; + } + + memory = default; + return false; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] ReadOnlyMemory GetFirstMemory() { diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 3e1bfad..c9fabd7 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -128,15 +128,24 @@ public ReadOnlyMemory CurrentBufferedBody get { Debug.Assert(_publicationState is PublicationState.Current); - if (_current.TryGetBufferedFirstMemory(0, out var body) + if (_current.TryGetBufferedArrayMemory(0, out var body) && body.Length == _current.Header.BodyLength) return body; - - var sequence = _current.GetSequence(); - return _current.GetContiguousMemory(sequence); + return GetCurrentBufferedBodySlow(); } } + [MethodImpl(MethodImplOptions.NoInlining)] + ReadOnlyMemory GetCurrentBufferedBodySlow() + { + if (_current.TryGetBufferedFirstMemory(0, out var body) + && body.Length == _current.Header.BodyLength) + return body; + + var sequence = _current.GetSequence(); + return _current.GetContiguousMemory(sequence); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGetCurrent(out BackendMessage current) { From 77a0d2b76ccbd41fee87ce7bcc21bcd139849fb9 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 14:13:25 +0200 Subject: [PATCH 021/136] Peel collected-row loop entry state --- .../Flows/CommandFlow.MessageEnumerator.cs | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 6becaba..bc99087 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -229,58 +229,65 @@ CollectRowsStatus CollectAvailableRows( out BackendMessage.Accessor pending, out BackendMessage terminal) { - while (true) + var decoder = _decoder; + if (!currentReady) { - if (!currentReady) + if (_first) { - if (_first) - { - _first = false; - currentReady = true; - } - else + _first = false; + } + else + { + _exceptionDispatchInfo?.Throw(); + if (_done) + ThrowHelper.ThrowInvalidOperation( + "Underlying message enumerator completed before a terminal message was returned."); + if (!decoder.TryMoveNext()) { - _exceptionDispatchInfo?.Throw(); - if (_done) - ThrowHelper.ThrowInvalidOperation( - "Underlying message enumerator completed before a terminal message was returned."); - if (!_decoder.TryMoveNext()) - { - pending = default; - terminal = default; - return CollectRowsStatus.RequiresInput; - } - currentReady = true; + pending = default; + terminal = default; + return CollectRowsStatus.RequiresInput; } } + } - DebugEnsureExpected(_decoder.Current); - if (_decoder.CurrentType is not PgTypes.BackendType.DataRow) + var collect = _collectorException is null; + while (true) + { + DebugEnsureExpected(decoder.Current); + if (decoder.CurrentType is not PgTypes.BackendType.DataRow) { _done = true; pending = default; - terminal = _decoder.Current; + terminal = decoder.Current; return CollectRowsStatus.Complete; } - if (!_decoder.CurrentBuffered) + if (!decoder.CurrentBuffered) { - pending = _decoder.CurrentAccessor; + pending = decoder.CurrentAccessor; terminal = default; return CollectRowsStatus.RequiresBuffer; } - if (_collectorException is null) + if (collect) { try { - collector(state, new CommandResult.RowView(_decoder.CurrentBufferedBody)); + collector(state, new CommandResult.RowView(decoder.CurrentBufferedBody)); } catch (Exception ex) { _collectorException = ExceptionDispatchInfo.Capture(ex); + collect = false; } } - currentReady = false; + + if (!decoder.TryMoveNext()) + { + pending = default; + terminal = default; + return CollectRowsStatus.RequiresInput; + } } } From 38e1232469e7b4e9e8a6e2348efff1c718f77907 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 14:35:53 +0200 Subject: [PATCH 022/136] Shape collected row advancement for the JIT --- Slon/Pg/Protocol/BackendMessageContext.cs | 9 +- Slon/Pg/Protocol/BackendMessageCursor.cs | 1 + .../Flows/CommandFlow.MessageEnumerator.cs | 90 +++++++++---------- Slon/Pg/Protocol/PgDecoder.cs | 7 +- Slon/Pg/Protocol/ProtocolReadPipe.cs | 1 + 5 files changed, 55 insertions(+), 53 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index c9fabd7..7e9574e 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -399,7 +399,7 @@ public bool TryMoveNext() { if (_publicationState is PublicationState.Peeked) { - _publicationState = PublicationState.Current; + PublishPeeked(); return true; } if (!_cursor.TryReadNextInPlace(out var header, out var buffer, out var bufferLength)) @@ -420,6 +420,13 @@ void ResetMessageState() } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PublishPeeked() + { + Debug.Assert(_publicationState is PublicationState.Peeked); + _publicationState = PublicationState.Current; + } + public void RetireCursor(bool retainProjections = false) { if (!retainProjections) diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index f59f029..7f2e399 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -159,6 +159,7 @@ public ReadOnlySpan FirstSpan }; // Returns the sequence before the index, stores the sequence after it in place. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public FastReadOnlySequence SplitInPlace(long offset) { var firstEnd = ReferenceEquals(_startObject, _endObject) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index bc99087..68f63b8 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -230,65 +230,57 @@ CollectRowsStatus CollectAvailableRows( out BackendMessage terminal) { var decoder = _decoder; - if (!currentReady) + var collect = _collectorException is null; + if (currentReady) + goto ProcessCurrent; + if (_first) { - if (_first) - { - _first = false; - } - else - { - _exceptionDispatchInfo?.Throw(); - if (_done) - ThrowHelper.ThrowInvalidOperation( - "Underlying message enumerator completed before a terminal message was returned."); - if (!decoder.TryMoveNext()) - { - pending = default; - terminal = default; - return CollectRowsStatus.RequiresInput; - } - } + _first = false; + goto ProcessCurrent; } - var collect = _collectorException is null; - while (true) + _exceptionDispatchInfo?.Throw(); + if (_done) + ThrowHelper.ThrowInvalidOperation( + "Underlying message enumerator completed before a terminal message was returned."); + + MoveNext: + if (!decoder.TryMoveNext()) { - DebugEnsureExpected(decoder.Current); - if (decoder.CurrentType is not PgTypes.BackendType.DataRow) - { - _done = true; - pending = default; - terminal = decoder.Current; - return CollectRowsStatus.Complete; - } - if (!decoder.CurrentBuffered) - { - pending = decoder.CurrentAccessor; - terminal = default; - return CollectRowsStatus.RequiresBuffer; - } + pending = default; + terminal = default; + return CollectRowsStatus.RequiresInput; + } + + ProcessCurrent: + DebugEnsureExpected(decoder.Current); + if (decoder.CurrentType is not PgTypes.BackendType.DataRow) + { + _done = true; + pending = default; + terminal = decoder.Current; + return CollectRowsStatus.Complete; + } + if (!decoder.CurrentBuffered) + { + pending = decoder.CurrentAccessor; + terminal = default; + return CollectRowsStatus.RequiresBuffer; + } - if (collect) + if (collect) + { + try { - try - { - collector(state, new CommandResult.RowView(decoder.CurrentBufferedBody)); - } - catch (Exception ex) - { - _collectorException = ExceptionDispatchInfo.Capture(ex); - collect = false; - } + collector(state, new CommandResult.RowView(decoder.CurrentBufferedBody)); } - - if (!decoder.TryMoveNext()) + catch (Exception ex) { - pending = default; - terminal = default; - return CollectRowsStatus.RequiresInput; + _collectorException = ExceptionDispatchInfo.Capture(ex); + collect = false; } } + goto MoveNext; } public void ThrowCollectorException() diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 43db1a5..73c189d 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -813,8 +813,9 @@ or PgTypes.BackendType.NoticeResponse or PgTypes.BackendType.NotificationResponse or PgTypes.BackendType.ParameterStatus)) { - var moved = TryMoveNext(_pipe); - Debug.Assert(moved); + _pipe.PublishPeeked(); + if (type is PgTypes.BackendType.ErrorResponse) + ObserveMessage(_pipe.Current); return true; } @@ -825,7 +826,7 @@ or PgTypes.BackendType.NotificationResponse { goto unavailable; } - TryMoveNext(_pipe); + _pipe.PublishPeeked(); if (handled) continue; return true; diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index f6a381b..df35802 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -38,6 +38,7 @@ public bool TryGetCurrent(out BackendMessage message) public bool TryMoveNext() => _messageContext.TryMoveNext(); public bool TryPeekNext(out BackendHeader header) => _messageContext.TryPeekNext(out header); + public void PublishPeeked() => _messageContext.PublishPeeked(); public BackendMessage Peeked => _messageContext.Peeked; public void BindDecoder(PgDecoder decoder) => _messageContext.BindDecoder(decoder); From 49d63d91707a53a345afbed228233fba9eaf09af Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 15:09:50 +0200 Subject: [PATCH 023/136] Outline uncommon decoder paths --- Slon/Pg/Protocol/BackendMessageCursor.cs | 15 +++++- Slon/Pg/Protocol/PgClientFlow.cs | 19 ++++--- Slon/Pg/Protocol/PgDecoder.cs | 68 +++++++++++++----------- 3 files changed, 61 insertions(+), 41 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index 7f2e399..a12f009 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Slon.Pipelines; using static Slon.Pg.Protocol.PgTypes; @@ -52,7 +53,7 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence MaxMessageLength) - throw new PgFramingException($"PostgreSQL backend message length {protoHeader.MessageLength} exceeds the maximum supported length."); + ThrowMessageTooLong(protoHeader.MessageLength); var required = backendType is BackendType.DataRow ? Math.Min(protoHeader.MessageLength, (uint)_dataRowStreamingThreshold) : protoHeader.MessageLength; @@ -74,6 +75,12 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence throw new PgFramingException( + $"PostgreSQL backend message length {messageLength} exceeds the maximum supported length."); + public readonly bool TryReadNext(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength, out BackendMessageCursor remaining) { var thisCopy = this; @@ -195,6 +202,12 @@ public FastReadOnlySequence SplitInPlace(long offset) return prev; } + return SplitSlow(offset); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + FastReadOnlySequence SplitSlow(long offset) + { var sequence = Sequence; var prefix = sequence.Slice(0, offset); var remaining = sequence.Slice(offset); diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 2fc4374..aaa01ed 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -755,6 +755,13 @@ public void OnMessageWrite(PgTypes.FrontendType type) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool ShouldHandle(PgTypes.BackendType type) + => type is PgTypes.BackendType.ReadyForQuery + or PgTypes.BackendType.NoticeResponse + or PgTypes.BackendType.NotificationResponse + or PgTypes.BackendType.ParameterStatus; + /// Try-shape sync attempt: returns true if the message was processed without I/O. handled is /// true if the protocol layer consumed it (caller skips and pulls the next), false if it /// should be surfaced to the flow. Returns false only when a handler genuinely needs async @@ -763,11 +770,7 @@ public void OnMessageWrite(PgTypes.FrontendType type) [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryHandleMessage(in BackendMessage backendMessage, out bool handled) { - if (backendMessage.Header.Type - is PgTypes.BackendType.ReadyForQuery - or PgTypes.BackendType.NoticeResponse - or PgTypes.BackendType.NotificationResponse - or PgTypes.BackendType.ParameterStatus) + if (ShouldHandle(backendMessage.Header.Type)) { return TryHandleMessageCore(backendMessage, out handled); } @@ -781,11 +784,7 @@ or PgTypes.BackendType.NotificationResponse [MethodImpl(MethodImplOptions.AggressiveInlining)] public ValueTask HandleMessageAuto(in BackendMessage backendMessage) { - return backendMessage.Header.Type - is PgTypes.BackendType.ReadyForQuery - or PgTypes.BackendType.NoticeResponse - or PgTypes.BackendType.NotificationResponse - or PgTypes.BackendType.ParameterStatus + return ShouldHandle(backendMessage.Header.Type) ? HandleMessageAutoCore(backendMessage) : new(false); } diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 73c189d..351bf5b 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -800,44 +800,52 @@ bool TryMoveNext(ProtocolReadPipe pipe) internal bool TryMoveNext() => TryMoveNextCore(); bool TryMoveNextCore() + { + if (!_pipe.TryPeekNext(out var header)) + { + PrepareRead(); + return false; + } + + var type = header.Type; + // Only auto-handled messages need the transactional peek slot: their handler may need + // I/O and decline the synchronous path. Ordinary messages can publish directly. + if (PgClientFlow.ExecutionControl.ShouldHandle(type)) + return TryMoveNextAutoHandled(type); + + _pipe.PublishPeeked(); + if (type is PgTypes.BackendType.ErrorResponse) + ObserveMessage(_pipe.Current); + return true; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + bool TryMoveNextAutoHandled(PgTypes.BackendType type) { while (true) { - while (_pipe.TryPeekNext(out var header)) + var handled = false; + if (type is PgTypes.BackendType.ReadyForQuery) + RestoreDefaultReadTimeout(); + if (!CurrentExecutionControl.TryHandleMessage(_pipe.Peeked, out handled)) + return false; + + _pipe.PublishPeeked(); + if (!handled) { - var type = header.Type; - // Only auto-handled messages need the transactional peek slot: their handler may need - // I/O and decline the synchronous path. Ordinary messages can publish directly. - if (type is not (PgTypes.BackendType.ReadyForQuery - or PgTypes.BackendType.NoticeResponse - or PgTypes.BackendType.NotificationResponse - or PgTypes.BackendType.ParameterStatus)) - { - _pipe.PublishPeeked(); - if (type is PgTypes.BackendType.ErrorResponse) - ObserveMessage(_pipe.Current); - return true; - } - - var handled = false; - if (type is PgTypes.BackendType.ReadyForQuery) - RestoreDefaultReadTimeout(); - if (!CurrentExecutionControl.TryHandleMessage(_pipe.Peeked, out handled)) - { - goto unavailable; - } - _pipe.PublishPeeked(); - if (handled) - continue; + if (type is PgTypes.BackendType.ErrorResponse) + ObserveMessage(_pipe.Current); return true; } - PrepareRead(); - break; - } + if (!_pipe.TryPeekNext(out var header)) + { + PrepareRead(); + return false; + } - unavailable: - return false; + type = header.Type; + } } // Auto-switch read, mirroring the encoder's FlushAuto: a sync flow takes the BLOCKING read path From d539765f47a5cc7906741c49966bc2f1886d2208 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 15:10:02 +0200 Subject: [PATCH 024/136] Separate row scanning from exit publication --- .../Flows/CommandFlow.MessageEnumerator.cs | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 68f63b8..5cbfd21 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -228,6 +228,31 @@ CollectRowsStatus CollectAvailableRows( bool currentReady, out BackendMessage.Accessor pending, out BackendMessage terminal) + { + var status = CollectAvailableRowsCore(state, collector, currentReady); + if (status is CollectRowsStatus.RequiresInput) + { + pending = default; + terminal = default; + } + else if (status is CollectRowsStatus.RequiresBuffer) + { + pending = _decoder.CurrentAccessor; + terminal = default; + } + else + { + _done = true; + pending = default; + terminal = _decoder.Current; + } + return status; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + CollectRowsStatus CollectAvailableRowsCore( + TState state, Action collector, + bool currentReady) { var decoder = _decoder; var collect = _collectorException is null; @@ -246,27 +271,14 @@ CollectRowsStatus CollectAvailableRows( MoveNext: if (!decoder.TryMoveNext()) - { - pending = default; - terminal = default; return CollectRowsStatus.RequiresInput; - } ProcessCurrent: DebugEnsureExpected(decoder.Current); if (decoder.CurrentType is not PgTypes.BackendType.DataRow) - { - _done = true; - pending = default; - terminal = decoder.Current; return CollectRowsStatus.Complete; - } if (!decoder.CurrentBuffered) - { - pending = decoder.CurrentAccessor; - terminal = default; return CollectRowsStatus.RequiresBuffer; - } if (collect) { From e30a6adde2e6ac488faf2a461c94171d9be261ab Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 19:28:24 +0200 Subject: [PATCH 025/136] Specialize suspended direct reads --- Slon/Pg/Protocol/PgDecoder.cs | 92 ++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 351bf5b..37a8dfd 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -504,7 +504,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau while (true) { if (!directReadTask.IsCompletedSuccessfully) - return MoveNextAsyncCore(null, directReadTask, null, cancellationToken, frontierFlow); + return MoveNextDirectAsync(directReadTask, cancellationToken, frontierFlow); if (CompleteDirectRead(directReadTask.Result, readToken, out directReadTask, out var readFinished, out var directReadCompleted)) @@ -557,6 +557,96 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau } + [MethodImpl(MethodImplOptions.NoInlining)] + async ValueTask MoveNextDirectAsync( + ValueTask directReadTask, + CancellationToken cancellationToken, + PgClientFlow frontierFlow) + { + var timeoutSet = false; + var registration = cancellationToken.UnsafeRegister( + static (state, _) => ((CancellationTokenSource)state!).Cancel(), + _cancellationTokenSource); + try + { + while (true) + { + try + { + if (!timeoutSet) + { + ArmReadTimeout(); + timeoutSet = true; + } + var length = await directReadTask.ConfigureAwait(false); + if (CompleteDirectRead(length, + _cancellationTokenSource.Token, + out var nextDirectRead, out var readFinished, + out var readCompleted)) + { + LeaveCancellationReadFrontier(frontierFlow); + frontierFlow = null!; + } + else if (!readFinished) + { + directReadTask = nextDirectRead; + continue; + } + else + { + LeaveCancellationReadFrontier(frontierFlow); + frontierFlow = null!; + if (readCompleted) + return ReadCompleted(); + } + } + catch (Exception ex) + { + AbortDirectRead(); + if (frontierFlow is not null) + { + LeaveCancellationReadFrontier(frontierFlow); + frontierFlow = null!; + } + if (_cancellationTokenSource.IsCancellationRequested) + throw TranslateReadCancellation(ex, cancellationToken); + if (ex is EndOfStreamException eof) + throw TranslateEof(eof); + throw; + } + + while (TryMoveNext(_pipe)) + { + var handleTask = CurrentExecutionControl.HandleMessageAuto(_pipe.Current); + if (!handleTask.IsCompletedSuccessfully) + { + if (!await handleTask.ConfigureAwait(false)) + return true; + continue; + } + if (!handleTask.Result) + return true; + } + + PrepareRead(); + var token = _cancellationTokenSource.Token; + frontierFlow = EnterCancellationReadFrontier(); + if (!TryBeginDirectRead(token, out directReadTask)) + return await MoveNextAsyncCore( + _pipe.ReadAsync(token), null, null, + cancellationToken, frontierFlow).ConfigureAwait(false); + } + } + finally + { + if (frontierFlow is not null) + LeaveCancellationReadFrontier(frontierFlow); + registration.Dispose(); + if (timeoutSet) + SetRemainingTimeout(Timeout.InfiniteTimeSpan); + } + } + [MethodImpl(MethodImplOptions.NoInlining)] async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTask? directReadTask, ValueTask? messageHandledTask, CancellationToken cancellationToken, PgClientFlow? frontierFlow = null) { From 7f2de0b6c661c11b674a1e197988c3c644976184 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 19:58:25 +0200 Subject: [PATCH 026/136] Make existing-pipeline admission explicit --- Slon/Pg/Protocol/PgClientProtocol.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index ffcd2f8..8fe2efb 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -624,10 +624,10 @@ public bool TryQueue(PgClientFlow flow, FlowEnqueueOptions options = FlowEnqueue if ((options & FlowEnqueueOptions.RequireExistingPipeline) != 0) { - if (!TryQueueFlow(flow, options, static protocol => protocol.PipelineDepth > 0, this)) + if (!TryQueueFlow(flow, options, requireExistingPipeline: true)) return false; } - else if (!TryQueueFlow(flow, options, null, (object?)null)) + else if (!TryQueueFlow(flow, options)) return false; try @@ -752,14 +752,14 @@ internal async ValueTask BeginExclusiveScopeAsync( return scope; } - bool TryQueueFlow(PgClientFlow flow, FlowEnqueueOptions options, - Func? predicate = null, TState state = default!) - => TryQueueFlow(flow, ProtocolStatus.Ready, options, predicate, state); + bool TryQueueFlow(PgClientFlow flow, FlowEnqueueOptions options, + bool requireExistingPipeline = false) + => TryQueueFlow(flow, ProtocolStatus.Ready, options, requireExistingPipeline); bool TryQueueFlow(PgClientFlow flow, ProtocolStatus requiredStatus) - => TryQueueFlow(flow, requiredStatus, FlowEnqueueOptions.None); - bool TryQueueFlow(PgClientFlow flow, ProtocolStatus requiredStatus, - FlowEnqueueOptions options, Func? predicate = null, TState state = default!) + => TryQueueFlow(flow, requiredStatus, FlowEnqueueOptions.None, requireExistingPipeline: false); + bool TryQueueFlow(PgClientFlow flow, ProtocolStatus requiredStatus, + FlowEnqueueOptions options, bool requireExistingPipeline) { // A handoff-capable sync flow is held at its FIFO turn. Consumer-driven flows defer the // handoff; self-driven flows take it as part of admission. @@ -772,7 +772,7 @@ bool TryQueueFlow(PgClientFlow flow, ProtocolStatus requiredStatus, (requiredStatus is ProtocolStatus.Ready && _admissionBlocked != 0)) return false; - if (predicate?.Invoke(state) == false) + if (requireExistingPipeline && _pipeline.Depth == 0) return false; if (requiredStatus is ProtocolStatus.Ready && From 34ac2a2ad3f8fd694c5153ff7cefe06acd69143d Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 20:07:47 +0200 Subject: [PATCH 027/136] Skip async result drain after exhaustion --- Slon/Pg/CommandResult.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index 82c2ed7..1437e4a 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -238,12 +238,18 @@ internal void Complete() EnsureComplete(); } - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] - internal async ValueTask CompleteAsync() + internal ValueTask CompleteAsync() { if (IsComplete) - return; + return default; + return CompleteAsyncCore(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] + async ValueTask CompleteAsyncCore() + { await _row.RevokeColumnLeaseAsync().ConfigureAwait(false); while (await MoveNextMessageAsync().ConfigureAwait(false)) { From 42b55641801fd91454e069a0b156d90d32d1fb66 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 20:20:24 +0200 Subject: [PATCH 028/136] Use a value marker for flow completion --- Slon/Pg/Protocol/PgClientFlow.cs | 20 +++++++++++--------- Slon/Pg/Protocol/PgClientProtocol.cs | 2 +- Slon/SlonConnection.cs | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index aaa01ed..df726c5 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -16,6 +16,8 @@ protected internal virtual void OnCompleted(PgClientFlow flow, Exception? except // them from the migratable source backlog, while standalone low-level flows remain initialized. abstract class PgClientFlowBindingContext; +readonly struct FlowCompletion; + sealed class FlowHandoffEvent : ManualResetEventSlim { PgClientFlowSource.State? _placementSource; @@ -74,7 +76,7 @@ internal void ResetInteraction() } [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] -public abstract class PgClientFlow : IValueTaskSource, IValueTaskSource, IThreadPoolWorkItem +public abstract class PgClientFlow : IValueTaskSource, IValueTaskSource, IThreadPoolWorkItem { PgClientProtocol.Control? _pendingActivationControl; FlowEnqueueOptions _enqueueOptions; @@ -166,7 +168,7 @@ void IThreadPoolWorkItem.Execute() // the flow types; the flow-as-result is the free disambiguator (the old Slon.Protocols // pattern). At most one pending waiter per tenure; post-completion awaits resolve // synchronously. - Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _completionCore; + Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _completionCore; int _completionClaim; ManualResetEventSlim? _completionEvent; // 1 while a WaitForComplete token is live (set at capture, cleared after GetResult consumed the @@ -299,7 +301,7 @@ void CompleteFlow(Exception? exception) if (Interlocked.CompareExchange(ref _completionClaim, 1, 0) != 0) return; if (exception is null) - _completionCore.SetResult(this, runContinuationsAsynchronously: true); + _completionCore.SetResult(default, runContinuationsAsynchronously: true); else _completionCore.SetException(exception, runContinuationsAsynchronously: true); } @@ -340,7 +342,7 @@ internal void DiscardUnqueued() // synchronously). The token is checked on entry only: the park itself is not cancelable, the // signal fires on every exit path (terminal, fault delivery, teardown), including the // cancel-delivered terminal. - internal ValueTask WaitForComplete(CancellationToken cancellationToken = default) + internal ValueTask WaitForComplete(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); // Publish waiter-pending before the token capture: a reuse path that observes the flag @@ -358,7 +360,7 @@ internal ValueTask WaitForComplete(CancellationToken cancellationT // Synchronous consumers must not block on an async continuation whose dispatch requires another // scheduler turn. The event is allocated only for that uncommon path and is signaled after the // completion core has published its result. - internal PgClientFlow WaitForCompleteSynchronously(CancellationToken cancellationToken = default) + internal void WaitForCompleteSynchronously(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); Volatile.Write(ref _completionWaiterPending, 1); @@ -374,7 +376,7 @@ internal PgClientFlow WaitForCompleteSynchronously(CancellationToken cancellatio if (_completionCore.GetStatus(token) is ValueTaskSourceStatus.Pending) completionEvent.Wait(); - return ((IValueTaskSource)this).GetResult(token); + _ = ((IValueTaskSource)this).GetResult(token); } /// True while a completion waiter holds an unconsumed token on this tenure's signal. Reuse @@ -463,7 +465,7 @@ protected virtual void OnReset() {} // bare flow ref. Keeps the handoff primitive off PgClientFlow's internal API, like _rfqCount. private protected virtual FlowHandoffEvent? HandoffEvent => null; - PgClientFlow IValueTaskSource.GetResult(short token) + FlowCompletion IValueTaskSource.GetResult(short token) { // Consume-then-clear: the release store orders the core consumption before the flag clear, // so a reuse path's acquire read of "not pending" proves the token's lifetime ended. Fault @@ -477,8 +479,8 @@ PgClientFlow IValueTaskSource.GetResult(short token) Volatile.Write(ref _completionWaiterPending, 0); } } - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _completionCore.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _completionCore.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _completionCore.OnCompleted(continuation, state, token, flags); PgDecoder IValueTaskSource.GetResult(short token) => _activationTaskSource.GetResult(token); diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index 8fe2efb..e6b789c 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -482,7 +482,7 @@ static void ReleaseTransportOnStartFailure(TransportConnection connection, Excep connection.Reader.Complete(reason); } - async ValueTask StartAsync(StartupFlow flow, ValueTask flowCompletion, CancellationToken cancellationToken = default) + async ValueTask StartAsync(StartupFlow flow, ValueTask flowCompletion, CancellationToken cancellationToken = default) { _source = PgClientFlowSource.Create( this, FlowControl, _executionScheduler, _options.MaxInFlightFlowsPerWire); diff --git a/Slon/SlonConnection.cs b/Slon/SlonConnection.cs index d991987..cad73f3 100644 --- a/Slon/SlonConnection.cs +++ b/Slon/SlonConnection.cs @@ -485,7 +485,7 @@ ValueTask CloseOwnedStatements(bool async, TrackedCommand[] tracked) _proxy.Enqueue(flow); return AwaitCompletion(completion); - static async ValueTask AwaitCompletion(ValueTask completion) + static async ValueTask AwaitCompletion(ValueTask completion) => _ = await completion.ConfigureAwait(false); } From 3a0def414934af3e585ecb3561412d56a3d1148d Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 20:23:59 +0200 Subject: [PATCH 029/136] Signal flow activation with a value marker --- Slon/Pg/Protocol/PgClientFlow.cs | 29 ++++++++++++++++------------ Slon/Pg/Protocol/PgClientProtocol.cs | 4 ++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index df726c5..e72f711 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -17,6 +17,7 @@ protected internal virtual void OnCompleted(PgClientFlow flow, Exception? except abstract class PgClientFlowBindingContext; readonly struct FlowCompletion; +readonly struct FlowActivation; sealed class FlowHandoffEvent : ManualResetEventSlim { @@ -76,7 +77,7 @@ internal void ResetInteraction() } [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] -public abstract class PgClientFlow : IValueTaskSource, IValueTaskSource, IThreadPoolWorkItem +public abstract class PgClientFlow : IValueTaskSource, IValueTaskSource, IThreadPoolWorkItem { PgClientProtocol.Control? _pendingActivationControl; FlowEnqueueOptions _enqueueOptions; @@ -176,7 +177,7 @@ void IThreadPoolWorkItem.Execute() int _completionWaiterPending; // Activation state. - Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _activationTaskSource; + Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _activationTaskSource; int _activationClaim; CancellationTokenRegistration _activationCancellationTokenRegistration; TimeSpan _remainingActivationTimeout; @@ -274,11 +275,11 @@ protected PgClientFlow(bool supportsDeferredFlush = false) _supportsDeferredFlush = supportsDeferredFlush; } - bool TrySetActivationResult(PgDecoder decoder, bool runContinuationsAsynchronously) + bool TrySetActivationResult(bool runContinuationsAsynchronously) { if (Interlocked.CompareExchange(ref _activationClaim, 1, 0) != 0) return false; - _activationTaskSource.SetResult(decoder, runContinuationsAsynchronously); + _activationTaskSource.SetResult(default, runContinuationsAsynchronously); return true; } @@ -483,9 +484,9 @@ FlowCompletion IValueTaskSource.GetResult(short token) void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _completionCore.OnCompleted(continuation, state, token, flags); - PgDecoder IValueTaskSource.GetResult(short token) => _activationTaskSource.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _activationTaskSource.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + FlowActivation IValueTaskSource.GetResult(short token) => _activationTaskSource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _activationTaskSource.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _activationTaskSource.OnCompleted(continuation, state, token, flags); @@ -892,11 +893,11 @@ public ValueTask ExecuteAuto() [MethodImpl(MethodImplOptions.NoInlining)] ValueTask ExecuteSynchronously() => flow.ExecuteAuto(new(this)); - public void Activate(PgDecoder decoder) + public void Activate() { flow._activationCancellationTokenRegistration.Dispose(); // If none of the cancellations triggered, we have a problem, throw. - if (!flow.TrySetActivationResult(decoder, runContinuationsAsynchronously: false) + if (!flow.TrySetActivationResult(runContinuationsAsynchronously: false) && !(flow._remainingActivationTimeout <= TimeSpan.Zero) && !control.AbortToken.IsCancellationRequested && !flow._activationCancellationTokenRegistration.Token.IsCancellationRequested) @@ -1096,17 +1097,21 @@ public bool IsDecoderReady public bool IsDecoderSettled => flow._activationTaskSource.GetStatus(flow._activationTaskSource.Version) is not ValueTaskSourceStatus.Pending; public PgDecoder GetDecoderResult() - => flow._activationTaskSource.GetResult(flow._activationTaskSource.Version); + { + _ = flow._activationTaskSource.GetResult(flow._activationTaskSource.Version); + return control.Decoder; + } public void OnDecoder(Action continuation, object? state, ValueTaskSourceOnCompletedFlags flags) => flow._activationTaskSource.OnCompleted(continuation, state, flow._activationTaskSource.Version, flags); // Bridge to Task for callers that need to block (sync flow body using // .GetAwaiter().GetResult()) or to compose with Task-based combinators. MVTSC has no // blocking GetResult of its own, so this is the only safe sync-wait path. - public Task GetDecoderTask(CancellationToken cancellationToken) + public async Task GetDecoderTask(CancellationToken cancellationToken) { RegisterActivationCancellation(cancellationToken); - return new ValueTask(flow, flow._activationTaskSource.Version).AsTask(); + await new ValueTask(flow, flow._activationTaskSource.Version).ConfigureAwait(false); + return control.Decoder; } // Registers caller cancellation against the activation source so a flow can unwind diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index e6b789c..eb3592b 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1612,7 +1612,7 @@ public void BindShells(PgDecoder decoder, ProtocolDataWriter writer) _writer = writer; } - PgDecoder Decoder => _decoder ?? protocol._pgDecoder; + internal PgDecoder Decoder => _decoder ?? protocol._pgDecoder; public PgClientFlow? ExecutingFlow => _slots.Executing; public PgClientFlow? ActivatedFlow => _slots.Activated; @@ -1825,7 +1825,7 @@ internal void BindDecoder(PgClientFlow flow) // off the executor via the TP dispatch. Safe to lag the flow's retirement: TrySetResult no-ops // on a flow the abort already faulted. internal void Activate(PgClientFlow flow) - => flow.GetExecutionControl(this).Activate(Decoder); + => flow.GetExecutionControl(this).Activate(); // Self-evict route for the flow layer's release-callback seam (see ExecutionControl.Release). internal void FailProtocol(Exception? reason) => protocol.FailProtocol(reason); From 6b02e17e2f53bb14a3e82b679c386d88aee6c281 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 20:27:29 +0200 Subject: [PATCH 030/136] Derive decoder heartbeat ownership from activation --- Slon/Pg/Protocol/PgClientFlow.cs | 10 +++------- Slon/Pg/Protocol/PgDecoder.cs | 6 +----- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index e72f711..ce2a60d 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -143,7 +143,6 @@ void IThreadPoolWorkItem.Execute() readonly bool _supportsDeferredFlush; internal bool SupportsDeferredFlush => _supportsDeferredFlush; - Action? _decoderOnHeartbeatAction; // TODO should we have this here? int _rfqCount; int _cancellationWindow; internal int CancellationWindow => Volatile.Read(ref _cancellationWindow); @@ -904,11 +903,6 @@ public void Activate() ThrowHelper.ThrowInvalidOperation("Flow was already activated unexpectedly."); } - public void RegisterDecoderOnHeartbeat(Action action) - { - flow._decoderOnHeartbeatAction = action; - } - public void OnHeartbeat(TimeSpan interval) { if (PropagateTermination()) @@ -916,7 +910,9 @@ public void OnHeartbeat(TimeSpan interval) OnActivationHeartbeat(interval); - flow._decoderOnHeartbeatAction?.Invoke(interval); + if (flow._activationTaskSource.GetStatus(flow._activationTaskSource.Version) + is ValueTaskSourceStatus.Succeeded) + control.Decoder.OnHeartbeat(interval); flow.OnHeartbeat(interval); } diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 37a8dfd..45a4721 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -23,7 +23,6 @@ public sealed class PgDecoder: IEnumerator, IAsyncEnumerator _onHeartbeatAction; CancellationTokenSource _cancellationTokenSource; TimeSpan _readTimeout; @@ -94,7 +93,6 @@ TimeSpan GetRemainingTimeout() _readTimeout = defaultReadTimeout; _readTimeoutArmed = readTimeoutArmed; _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(abortToken); - _onHeartbeatAction = OnHeartbeat; SetRemainingTimeout(Timeout.InfiniteTimeSpan); } @@ -223,8 +221,6 @@ internal void Initialize(PgClientProtocol.Control control) if (!ReferenceEquals(_control, control)) _control = control; _pipe.BindDecoder(this); - // TODO we want a heartbeat setup directly through the protocol on construction. - CurrentExecutionControl.RegisterDecoderOnHeartbeat(_onHeartbeatAction); } /// @@ -373,7 +369,7 @@ internal CurrentMessageBuffer ExtendCurrentMessage() } } - void OnHeartbeat(TimeSpan elapsed) + internal void OnHeartbeat(TimeSpan elapsed) { var ticks = Interlocked.Exchange(ref _remainingTimeoutTicks, ClaimedTimeoutTicks); if (ticks == ClaimedTimeoutTicks) From a966164aab23b77594e8151dbd75715d4c2ab5e7 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 21:43:01 +0200 Subject: [PATCH 031/136] Reuse buffered length for stream read sizing --- Slon/Pipelines/StreamPipeReader.cs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Slon/Pipelines/StreamPipeReader.cs b/Slon/Pipelines/StreamPipeReader.cs index bbe35c1..007ae3e 100644 --- a/Slon/Pipelines/StreamPipeReader.cs +++ b/Slon/Pipelines/StreamPipeReader.cs @@ -336,13 +336,7 @@ protected ReadResult ReadCore(int minimumSize, TimeSpan timeout) do { // We know minimumSize must be null or larger than what we have buffered to get here. - var segmentSize = minimumSize; - if (minimumSize is not 0) - { - // We must request a segment that is minimumSize - BufferedBytes. - Debug.Assert(Segments.BufferedBytes <= minimumSize); - segmentSize -= (int)Segments.BufferedBytes; - } + var segmentSize = GetReadSizeHint(minimumSize); // We don't mind if we get smaller segments, we just want to make progress towards minimumSize. var buffer = Segments.Reserve(segmentSize, enforceHint: false); @@ -426,12 +420,7 @@ async ValueTask ReadAsyncCore(int minimumSize, int length; do { - var segmentSize = minimumSize; - if (minimumSize is not 0) - { - Debug.Assert(Segments.BufferedBytes <= minimumSize); - segmentSize -= (int)Segments.BufferedBytes; - } + var segmentSize = GetReadSizeHint(minimumSize); var buffer = Segments.Reserve(segmentSize, enforceHint: false); length = await Stream.ReadAsync(buffer, token).ConfigureAwait(false); @@ -466,6 +455,17 @@ async ValueTask ReadAsyncCore(int minimumSize, } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + int GetReadSizeHint(int minimumSize) + { + if (minimumSize is 0) + return 0; + var bufferedBytes = Segments.BufferedBytes; + return bufferedBytes < minimumSize + ? minimumSize - (int)bufferedBytes + : 0; + } + protected async Task CopyToAsyncCore(PipeWriter destination, CancellationToken cancellationToken = default) { if (!TryStartRead()) From 5c37639429ff5cd09e1f557d5b32bbcd6f279f75 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 21:52:49 +0200 Subject: [PATCH 032/136] Specialize protocol reads for Runtime Async on net11 --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 2 + Slon.Tests/Pg/PostgreSqlSslTests.cs | 4 +- Slon.Tests/Slon.Tests.csproj | 3 +- Slon/Pg/Protocol/PgDecoder.cs | 60 +++++++++++++++++-- Slon/Pipelines/StreamPipeReader.cs | 27 ++++++--- Slon/Slon.csproj | 3 +- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index ca5e5e8..bc0940d 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -340,6 +340,7 @@ public async Task RepeatedQueryFrames_WithSmallRecycledBuffers_NeverEnterMessage await readPipe.DisposeAsync(); } +#if !NET11_0_OR_GREATER [TestMethod] public async Task RepeatedQueryFrames_ThroughDirectReads_NeverEnterMessageBodies() { @@ -405,6 +406,7 @@ void ValidateMessages() } } } +#endif static byte[][] QueryResponseBytes() { diff --git a/Slon.Tests/Pg/PostgreSqlSslTests.cs b/Slon.Tests/Pg/PostgreSqlSslTests.cs index 42e63e0..039245e 100644 --- a/Slon.Tests/Pg/PostgreSqlSslTests.cs +++ b/Slon.Tests/Pg/PostgreSqlSslTests.cs @@ -175,7 +175,7 @@ public async Task VerifyFull_RejectsUntrustedCertificate() await CreateFactory(listener, PostgreSqlSslNegotiation.Direct, PostgreSqlSslMode.VerifyFull).CreateAsync()); try { await server; } - catch (IOException) { } + catch (Exception ex) when (ex is IOException or AuthenticationException) { } } [TestMethod] @@ -190,7 +190,7 @@ public async Task VerifyCA_RejectsUntrustedCertificate() await CreateFactory(listener, PostgreSqlSslNegotiation.Direct, PostgreSqlSslMode.VerifyCA).CreateAsync()); try { await server; } - catch (IOException) { } + catch (Exception ex) when (ex is IOException or AuthenticationException) { } } [TestMethod] diff --git a/Slon.Tests/Slon.Tests.csproj b/Slon.Tests/Slon.Tests.csproj index a1dbcc8..5273122 100644 --- a/Slon.Tests/Slon.Tests.csproj +++ b/Slon.Tests/Slon.Tests.csproj @@ -1,6 +1,7 @@ - net10.0 + net10.0;net11.0 + runtime-async=on preview enable enable diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 45a4721..2df7570 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -19,7 +19,9 @@ namespace Slon.Pg.Protocol; public sealed class PgDecoder: IEnumerator, IAsyncEnumerator { readonly ProtocolReadPipe _pipe; +#if !NET11_0_OR_GREATER readonly StreamPipeReader? _directReader; +#endif readonly CancellationToken _abortToken; readonly TimeSpan _defaultReadTimeout; readonly Action? _readTimeoutArmed; @@ -87,7 +89,9 @@ TimeSpan GetRemainingTimeout() Action? readTimeoutArmed) { _pipe = pipe; +#if !NET11_0_OR_GREATER _directReader = pipe.PipeReader as StreamPipeReader; +#endif _abortToken = abortToken; _defaultReadTimeout = defaultReadTimeout; _readTimeout = defaultReadTimeout; @@ -172,6 +176,7 @@ bool ReadNext(TimeSpan timeout) return _pipe.MoveNext(timeout); } +#if !NET11_0_OR_GREATER bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask task) { if (_directReader is { SupportsDirectRead: true } directReader) @@ -197,6 +202,7 @@ bool CompleteDirectRead(int length, CancellationToken cancellationToken, } void AbortDirectRead() => _directReader!.AbortDirectRead(); +#endif // Builds a scope-bound shell over the shared pipe with the scope's abort token. internal static PgDecoder CreateScopeShell(PgDecoder baseShell, CancellationToken abortToken, TimeSpan defaultReadTimeout) => new(baseShell._pipe, abortToken, defaultReadTimeout, baseShell._readTimeoutArmed); @@ -471,6 +477,7 @@ internal bool IsAtCancellationReadFrontier(PgClientFlow flow, int window) /// Flow-owned cancellation path for a parked read. Without it the only break-out is protocol /// abort. An uncaught firing triggers the protocol's recovery path, so prefer a /// coordination-boundary check in connection-preserving flows. +#if !NET11_0_OR_GREATER public ValueTask MoveNextAsync(CancellationToken cancellationToken = default) { EnsureUsableCts(); @@ -643,15 +650,28 @@ async ValueTask MoveNextDirectAsync( } } +#endif +#if NET11_0_OR_GREATER + public async ValueTask MoveNextAsync(CancellationToken cancellationToken = default) + { + EnsureUsableCts(); + var pipe = _pipe; + PgClientFlow? frontierFlow = null; +#else [MethodImpl(MethodImplOptions.NoInlining)] - async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTask? directReadTask, ValueTask? messageHandledTask, CancellationToken cancellationToken, PgClientFlow? frontierFlow = null) + async ValueTask MoveNextAsyncCore(ValueTask? readTask, + ValueTask? directReadTask, ValueTask? messageHandledTask, + CancellationToken cancellationToken, PgClientFlow? frontierFlow = null) { + var pipe = _pipe; +#endif var timeoutSet = false; var registration = cancellationToken.UnsafeRegister(static (state, _) => ((CancellationTokenSource)state!).Cancel(), _cancellationTokenSource); try { while (true) { +#if !NET11_0_OR_GREATER if (messageHandledTask is { } t) { if (!await t.ConfigureAwait(false)) @@ -688,7 +708,9 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa throw TranslateEof(ex); } } +#endif +#if !NET11_0_OR_GREATER if (directReadTask is { } pendingDirectRead) { try @@ -735,10 +757,15 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa throw; } } +#endif - while (TryMoveNext(_pipe)) + while (TryMoveNext(pipe)) { - var handleTask = CurrentExecutionControl.HandleMessageAuto(_pipe.Current); + var handleTask = CurrentExecutionControl.HandleMessageAuto(pipe.Current); +#if NET11_0_OR_GREATER + if (!await handleTask.ConfigureAwait(false)) + return true; +#else if (!handleTask.IsCompletedSuccessfully) { messageHandledTask = handleTask; @@ -746,9 +773,12 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa } if (!handleTask.Result) return true; +#endif } +#if !NET11_0_OR_GREATER if (messageHandledTask.HasValue) continue; +#endif PrepareRead(); @@ -756,13 +786,33 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa { var token = _cancellationTokenSource.Token; frontierFlow = EnterCancellationReadFrontier(); +#if NET11_0_OR_GREATER + var nextRead = pipe.ReadAsync(token); + if (!nextRead.IsCompletedSuccessfully && !timeoutSet) + { + ArmReadTimeout(); + timeoutSet = true; + } + var result = await nextRead.ConfigureAwait(false); + LeaveCancellationReadFrontier(frontierFlow); + frontierFlow = null; + if (CompleteRead(result, token, out var readCompleted)) + continue; + if (readCompleted) + return ReadCompleted(); +#else if (TryBeginDirectRead(token, out var nextDirectRead)) directReadTask = nextDirectRead; else - readTask = _pipe.ReadAsync(token); + readTask = pipe.ReadAsync(token); +#endif } catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) { throw TranslateReadCancellation(ex, cancellationToken); } +#if NET11_0_OR_GREATER + catch (EndOfStreamException ex) + { throw TranslateEof(ex); } +#endif } } finally @@ -774,7 +824,9 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTa SetRemainingTimeout(Timeout.InfiniteTimeSpan); } } +#if !NET11_0_OR_GREATER } +#endif bool ReadCompleted() { diff --git a/Slon/Pipelines/StreamPipeReader.cs b/Slon/Pipelines/StreamPipeReader.cs index 007ae3e..eee1ee5 100644 --- a/Slon/Pipelines/StreamPipeReader.cs +++ b/Slon/Pipelines/StreamPipeReader.cs @@ -12,12 +12,14 @@ interface IStreamOwner : IDisposable, IAsyncDisposable { } abstract class StreamPipeReader : PipeReader { +#if !NET11_0_OR_GREATER readonly ValueTaskSourcePromise _readAsyncCorePromise = new(); + bool _directReadAwaitingData; +#endif readonly IStreamOwner? _streamOwner; int _isReadActive; int _readerCompleted; int _readerDisposed; - bool _directReadAwaitingData; // Null in direct-read mode (CancelPendingRead unsupported): the caller's token threads straight to // the underlying stream read, so neither this source nor a per-read registration is allocated. @@ -223,18 +225,19 @@ public override bool TryRead(out ReadResult result) return TryReadCore(out result); } - // Direct reads retain reader tenure until their continuation returns through CompleteDirectRead - // or AbortDirectRead. The protocol must interrupt and join that tenure before completing the - // reader and returning its destination buffer. - internal bool SupportsDirectRead => PendingReadTokenSource is null; - const int BufferedDirectRead = -1; - internal void EnsureCanUpgradeStream() { if (IsReaderCompleted || Volatile.Read(ref _isReadActive) is not 0 || Segments.BufferedBytes is not 0) throw new InvalidOperationException("The reader must be open, idle, and empty before its stream can be upgraded."); } +#if !NET11_0_OR_GREATER + // Direct reads retain reader tenure until their continuation returns through CompleteDirectRead + // or AbortDirectRead. The protocol must interrupt and join that tenure before completing the + // reader and returning its destination buffer. + internal bool SupportsDirectRead => PendingReadTokenSource is null; + const int BufferedDirectRead = -1; + // Direct leaf handoff. The caller awaits the stream's ValueTask directly, then returns // the byte count through CompleteDirectRead. This keeps buffer ownership and PipeReader read tenure // here while removing the intermediate ReadAsyncCore completion frame. @@ -300,12 +303,12 @@ internal void AbortDirectRead() if (Volatile.Read(ref _isReadActive) is not 0) EndStartedRead(); } - ValueTask StartDataRead(CancellationToken cancellationToken) { var buffer = Segments.Reserve(0, enforceHint: false); return Stream.ReadAsync(buffer, cancellationToken); } +#endif protected ReadResult ReadCore(int minimumSize, TimeSpan timeout) { @@ -378,6 +381,11 @@ protected ReadResult ReadCore(int minimumSize, TimeSpan timeout) } +#if NET11_0_OR_GREATER + protected async ValueTask ReadAsyncCore(int minimumSize, CancellationToken cancellationToken) + { + var tokenSource = PendingReadTokenSource; +#else protected ValueTask ReadAsyncCore(int minimumSize, CancellationToken cancellationToken) { PromiseAsyncValueTaskMethodBuilder.Promise = _readAsyncCorePromise; @@ -395,6 +403,7 @@ protected ValueTask ReadAsyncCore(int minimumSize, CancellationToken async ValueTask ReadAsyncCore(int minimumSize, AutoResetCancellationTokenSource? tokenSource, CancellationToken cancellationToken) { +#endif // Cancellation token was already checked before getting here. if (!TryStartRead()) ThrowAlreadyReading(); @@ -453,7 +462,9 @@ async ValueTask ReadAsyncCore(int minimumSize, } } } +#if !NET11_0_OR_GREATER } +#endif [MethodImpl(MethodImplOptions.AggressiveInlining)] int GetReadSizeHint(int minimumSize) diff --git a/Slon/Slon.csproj b/Slon/Slon.csproj index f7301cc..bac62fb 100644 --- a/Slon/Slon.csproj +++ b/Slon/Slon.csproj @@ -1,6 +1,7 @@ - net10.0 + net10.0;net11.0 + runtime-async=on enable enable true From a6c7a731200c4805ec0b47894686c07c476656f7 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 22:10:17 +0200 Subject: [PATCH 033/136] Outline contiguous projection release --- Slon/Pg/Protocol/BackendMessageContext.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 7e9574e..aad4a7d 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -240,15 +240,25 @@ public ReadOnlyMemory GetContiguousMemory( return buffer.AsMemory(0, length); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ReleaseContiguousProjections() { var projection = _contiguousProjections; + if (projection is null) + return; + ReleaseContiguousProjectionsCore(projection); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + void ReleaseContiguousProjectionsCore(ContiguousProjection projection) + { _contiguousProjections = null; - while (projection is not null) + do { ArrayPool.Shared.Return(projection.Buffer); - projection = projection.Next; + projection = projection.Next!; } + while (projection is not null); } public void BindDecoder(PgDecoder decoder) From 874411a57032dcee68506f2318fc59685443edda Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 22:42:05 +0200 Subject: [PATCH 034/136] Acquire manual-reset completion status --- Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs b/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs index ddd2004..25ca297 100644 --- a/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs +++ b/Slon/Threading/Tasks/Sources/ManualResetValueTaskSourceCore.cs @@ -116,7 +116,7 @@ public ValueTaskSourceStatus GetStatus(short token) ThrowInvalidOperationException(); } return - _continuation is null || !_completed ? ValueTaskSourceStatus.Pending : + _continuation is null || !Volatile.Read(ref _completed) ? ValueTaskSourceStatus.Pending : _error is null ? ValueTaskSourceStatus.Succeeded : _error.SourceException is OperationCanceledException ? ValueTaskSourceStatus.Canceled : ValueTaskSourceStatus.Faulted; From e936fd5498ae940fb67a3b76e12a492de2ad1d89 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 22:44:12 +0200 Subject: [PATCH 035/136] Skip result callback delivery when unobserved --- Slon/Pg/CommandResult.cs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index 1437e4a..fdaa268 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -385,21 +385,24 @@ void CompleteCommand(BackendMessage message) break; } - InvokeCompletionAction(); + if (_completionAction is not null) + InvokeCompletionAction(); } + [MethodImpl(MethodImplOptions.NoInlining)] void InvokeCompletionAction() { - if (_completionAction is { } action) + var action = _completionAction!; + var state = _completionActionState; + _completionAction = null; + _completionActionState = null; + try { - var state = _completionActionState; - _completionAction = null; - _completionActionState = null; - try { action(this, state); } - catch (Exception ex) - { - _flow.Fail(ex); - } + action(this, state); + } + catch (Exception ex) + { + _flow.Fail(ex); } } From 140cd98d51ab6366d26203048376cac4e17a2b7c Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 22:47:46 +0200 Subject: [PATCH 036/136] Avoid reclassifying known protocol messages --- Slon/Pg/Protocol/PgClientFlow.cs | 4 ++-- Slon/Pg/Protocol/PgDecoder.cs | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index ce2a60d..9f64653 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -774,7 +774,7 @@ public bool TryHandleMessage(in BackendMessage backendMessage, out bool handled) { if (ShouldHandle(backendMessage.Header.Type)) { - return TryHandleMessageCore(backendMessage, out handled); + return TryHandleKnownMessage(backendMessage, out handled); } handled = false; return true; @@ -791,7 +791,7 @@ public ValueTask HandleMessageAuto(in BackendMessage backendMessage) } [MethodImpl(MethodImplOptions.NoInlining)] - bool TryHandleMessageCore(BackendMessage backendMessage, out bool handled) + internal bool TryHandleKnownMessage(BackendMessage backendMessage, out bool handled) { switch (backendMessage.Header.Type) { diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 2df7570..510ceba 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -962,10 +962,11 @@ bool TryMoveNextAutoHandled(PgTypes.BackendType type) { while (true) { + Debug.Assert(PgClientFlow.ExecutionControl.ShouldHandle(type)); var handled = false; if (type is PgTypes.BackendType.ReadyForQuery) RestoreDefaultReadTimeout(); - if (!CurrentExecutionControl.TryHandleMessage(_pipe.Peeked, out handled)) + if (!CurrentExecutionControl.TryHandleKnownMessage(_pipe.Peeked, out handled)) return false; _pipe.PublishPeeked(); @@ -983,6 +984,13 @@ bool TryMoveNextAutoHandled(PgTypes.BackendType type) } type = header.Type; + if (!PgClientFlow.ExecutionControl.ShouldHandle(type)) + { + _pipe.PublishPeeked(); + if (type is PgTypes.BackendType.ErrorResponse) + ObserveMessage(_pipe.Current); + return true; + } } } From ac1301f105094a5b2677678d2af1d614d53772d5 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 23:11:40 +0200 Subject: [PATCH 037/136] Keep caller cancellation off direct read state --- Slon/Pg/Protocol/PgDecoder.cs | 43 ++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 510ceba..2d288fa 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -507,7 +507,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau while (true) { if (!directReadTask.IsCompletedSuccessfully) - return MoveNextDirectAsync(directReadTask, cancellationToken, frontierFlow); + return AwaitDirectRead(directReadTask, cancellationToken, frontierFlow); if (CompleteDirectRead(directReadTask.Result, readToken, out directReadTask, out var readFinished, out var directReadCompleted)) @@ -561,15 +561,20 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau [MethodImpl(MethodImplOptions.NoInlining)] - async ValueTask MoveNextDirectAsync( + ValueTask AwaitDirectRead( ValueTask directReadTask, CancellationToken cancellationToken, PgClientFlow frontierFlow) + => cancellationToken.CanBeCanceled + ? MoveNextDirectWithCancellationAsync(directReadTask, cancellationToken, frontierFlow) + : MoveNextDirectAsync(directReadTask, frontierFlow); + + [MethodImpl(MethodImplOptions.NoInlining)] + async ValueTask MoveNextDirectAsync( + ValueTask directReadTask, + PgClientFlow frontierFlow) { var timeoutSet = false; - var registration = cancellationToken.UnsafeRegister( - static (state, _) => ((CancellationTokenSource)state!).Cancel(), - _cancellationTokenSource); try { while (true) @@ -612,7 +617,7 @@ async ValueTask MoveNextDirectAsync( frontierFlow = null!; } if (_cancellationTokenSource.IsCancellationRequested) - throw TranslateReadCancellation(ex, cancellationToken); + throw TranslateReadCancellation(ex, default); if (ex is EndOfStreamException eof) throw TranslateEof(eof); throw; @@ -637,19 +642,41 @@ async ValueTask MoveNextDirectAsync( if (!TryBeginDirectRead(token, out directReadTask)) return await MoveNextAsyncCore( _pipe.ReadAsync(token), null, null, - cancellationToken, frontierFlow).ConfigureAwait(false); + default, frontierFlow).ConfigureAwait(false); } } finally { if (frontierFlow is not null) LeaveCancellationReadFrontier(frontierFlow); - registration.Dispose(); if (timeoutSet) SetRemainingTimeout(Timeout.InfiniteTimeSpan); } } + [MethodImpl(MethodImplOptions.NoInlining)] + async ValueTask MoveNextDirectWithCancellationAsync( + ValueTask directReadTask, + CancellationToken cancellationToken, + PgClientFlow frontierFlow) + { + var registration = cancellationToken.UnsafeRegister( + static (state, _) => ((CancellationTokenSource)state!).Cancel(), + _cancellationTokenSource); + try + { + return await MoveNextDirectAsync(directReadTask, frontierFlow).ConfigureAwait(false); + } + catch (Exception) when (cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + finally + { + registration.Dispose(); + } + } + #endif #if NET11_0_OR_GREATER public async ValueTask MoveNextAsync(CancellationToken cancellationToken = default) From f3d91e6b31ac9e4ca3df6096ba976439f1fa8733 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 23:21:03 +0200 Subject: [PATCH 038/136] Keep backend cursor parsing out of decoder callers --- Slon/Pg/Protocol/BackendMessageCursor.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index a12f009..1127f5b 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -40,6 +40,7 @@ public readonly BackendMessageCursor Slice(long offset) _dataRowStreamingThreshold, _initialLength); } + [MethodImpl(MethodImplOptions.NoInlining)] public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) { if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) && !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader)) From aa1d5342fcb91b5365f95d07ae075eb32a3307d7 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Wed, 2 Sep 2026 23:25:54 +0200 Subject: [PATCH 039/136] Keep message peek publication out of decoder callers --- Slon/Pg/Protocol/BackendMessageContext.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index aad4a7d..d680f01 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -487,6 +487,7 @@ public bool TryGetCursorUnread(out SequencePosition unread) // slot and the follow-up TryMoveNext picks it up without re-parsing. The returned // BackendMessage is valid until the next TryMoveNext (which bumps the version token); // use it immediately, don't store it. + [MethodImpl(MethodImplOptions.NoInlining)] public bool TryPeekNext(out BackendHeader header) { if (_publicationState is PublicationState.Peeked) From 2b5e817619f01342f7ca14f43ba5e77dfb5c2925 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 00:06:23 +0200 Subject: [PATCH 040/136] Use array-backed message storage directly --- Slon/Pg/Protocol/BackendMessage.cs | 26 +++++++++++++++++++++++ Slon/Pg/Protocol/BackendMessageContext.cs | 10 +++++++++ Slon/Pg/Protocol/BackendMessageCursor.cs | 9 +++++++- Slon/Pg/Row.cs | 12 ++++++++++- 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index c5c7ac9..71c12ed 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -242,6 +242,29 @@ internal bool TryGetBufferedArrayMemory(int offset, out ReadOnlyMemory mem return false; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGetBufferedArray(int offset, [NotNullWhen(true)] out byte[]? array, + out int arrayOffset, out int length) + { + Debug.Assert(Buffered); + offset += BackendHeader.ByteCount; + var firstLength = IsIndependent + ? _endIndexOrBufferedLength - _startIndex + : _endIndexOrBufferedLength; + if (_firstObject is byte[] value && (uint)offset <= (uint)firstLength) + { + array = value; + arrayOffset = _startIndex + offset; + length = firstLength - offset; + return true; + } + + array = null; + arrayOffset = 0; + length = 0; + return false; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] ReadOnlyMemory GetFirstMemory() { @@ -328,6 +351,9 @@ internal BackendMessageBodyReader OpenBodyReader() => _context.OpenCurrentBodyReader(_token); internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory memory) => _context.TryGetCurrentBufferedFirstMemory(_token, offset, out memory); + internal bool TryGetBufferedArray(int offset, [NotNullWhen(true)] out byte[]? array, + out int arrayOffset, out int length) + => _context.TryGetCurrentBufferedArray(_token, offset, out array, out arrayOffset, out length); internal void BufferBody() => _context.BufferCurrentMessage(_token); internal ValueTask BufferBodyAsync(CancellationToken cancellationToken) => _context.BufferCurrentMessageAsync(_token, cancellationToken); diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index d680f01..d882311 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Slon.Pipelines; @@ -175,6 +176,15 @@ internal bool TryGetCurrentBufferedFirstMemory(short token, int offset, return _current.TryGetBufferedFirstMemory(offset, out memory); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGetCurrentBufferedArray(short token, int offset, + [NotNullWhen(true)] out byte[]? array, + out int arrayOffset, out int length) + { + Validate(token); + return _current.TryGetBufferedArray(offset, out array, out arrayOffset, out length); + } + internal void SetCurrentFallbackBuffer( in ReadOnlySequence buffer, bool required) { diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index 1127f5b..2b6b8e2 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -150,7 +150,14 @@ public ReadOnlySpan FirstSpan { if (_startObject is null) return default; - var memory = FirstMemory; + if (_startObject is T[] array) + { + Debug.Assert(ReferenceEquals(_startObject, _endObject)); + return array.AsSpan(_startIndex, _endIndex - _startIndex); + } + var memory = _startObject is MemoryManager manager + ? manager.Memory + : ((ReadOnlySequenceSegment)_startObject).Memory; var end = ReferenceEquals(_startObject, _endObject) ? _endIndex : memory.Length; diff --git a/Slon/Pg/Row.cs b/Slon/Pg/Row.cs index 5f3f88b..d18b5cf 100644 --- a/Slon/Pg/Row.cs +++ b/Slon/Pg/Row.cs @@ -637,7 +637,17 @@ void CaptureBufferedBody() void CaptureBufferedBody(in BackendMessage.Accessor message) { - if (_bodyReader is null && message.TryGetBufferedFirstMemory(0, out var body)) + if (_bodyReader is null + && message.TryGetBufferedArray(0, out var array, out var offset, out var length)) + { + if (!ReferenceEquals(_bufferedArray, array)) + _bufferedArray = array; + _bufferedOffset = offset; + _bufferedLength = length; + if (!_bufferedBody.IsEmpty) + _bufferedBody = default; + } + else if (_bodyReader is null && message.TryGetBufferedFirstMemory(0, out var body)) { if (MemoryMarshal.TryGetArray(body, out var segment)) { From 75f7ce74d063d88e56663db0277465f6199cfe0d Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 00:23:03 +0200 Subject: [PATCH 041/136] Keep parsed message buffers scalar --- Slon/Pg/Protocol/BackendMessage.cs | 19 +++++++++++++++ Slon/Pg/Protocol/BackendMessageContext.cs | 29 +++++++++++++++++++---- Slon/Pg/Protocol/BackendMessageCursor.cs | 26 ++++++++++++++++---- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index 71c12ed..b8b79c2 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -73,6 +73,25 @@ internal static void Initialize(ref BackendMessage destination, BackendHeader he WriteGranularly(ref destination, in value, destinationIsZero: false); } + internal static void Initialize(ref BackendMessage destination, BackendHeader header, + in BackendMessageCursor.FastReadOnlySequence buffer, + BackendMessageContext context, short token, bool buffered) + { + var firstObject = buffer.StartObject; + if (!ReferenceEquals(destination._contextOrEndObject, context)) + Unsafe.AsRef(in destination._contextOrEndObject) = context; + if (!ReferenceEquals(destination._firstObject, firstObject)) + Unsafe.AsRef(in destination._firstObject) = firstObject; + + Unsafe.AsRef(in destination._state) = (buffered ? 1u : 0) + | ((uint)(byte)header.Type << 2) + | ((uint)(ushort)token << 10); + Unsafe.AsRef(in destination._length) = header.Length; + Unsafe.AsRef(in destination._startIndex) = buffer.StartIndex; + Unsafe.AsRef(in destination._endIndexOrBufferedLength) = checked((int)buffer.Length); + context.SetCurrentFallbackBuffer(in buffer); + } + // The JIT should have a phase for picking granular writes (and write barriers) over full struct assignments. // This translation is entirely mechanical (even though these implementations need to deviate for external types). [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index d882311..a6f10ff 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -39,12 +39,19 @@ public void Set(in ReadOnlySequence buffer) { var start = (ReadOnlySequenceSegment)buffer.Start.GetObject()!; var end = (ReadOnlySequenceSegment)buffer.End.GetObject()!; + Set(start, buffer.Start.GetInteger() & int.MaxValue, + end, buffer.End.GetInteger() & int.MaxValue); + } + + public void Set(ReadOnlySequenceSegment start, int startIndex, + ReadOnlySequenceSegment end, int endIndex) + { if (!ReferenceEquals(_start, start)) _start = start; if (!ReferenceEquals(_end, end)) _end = end; - _startIndex = buffer.Start.GetInteger() & int.MaxValue; - _endIndex = buffer.End.GetInteger() & int.MaxValue; + _startIndex = startIndex; + _endIndex = endIndex; } public void Clear() @@ -194,6 +201,20 @@ internal void SetCurrentFallbackBuffer( _currentFallbackBuffer.Clear(); } + internal void SetCurrentFallbackBuffer( + in BackendMessageCursor.FastReadOnlySequence buffer) + { + if (buffer.StartObject is ReadOnlySequenceSegment start) + { + _currentFallbackBuffer.Set(start, buffer.StartIndex, + (ReadOnlySequenceSegment)buffer.EndObject!, buffer.EndIndex); + } + else if (!_currentFallbackBuffer.IsEmpty) + { + _currentFallbackBuffer.Clear(); + } + } + internal ReadOnlySequence GetFallbackBuffer(short token) { Validate(token); @@ -422,7 +443,7 @@ public bool TryMoveNext() PublishPeeked(); return true; } - if (!_cursor.TryReadNextInPlace(out var header, out var buffer, out var bufferLength)) + if (!_cursor.TryReadNextBuffer(out var header, out var buffer, out var bufferLength)) return false; ResetMessageState(); if (bufferLength < header.MessageLength) @@ -505,7 +526,7 @@ public bool TryPeekNext(out BackendHeader header) header = _current.Header; return true; } - if (!_cursor.TryReadNextInPlace( + if (!_cursor.TryReadNextBuffer( out header, out var buffer, out var bufferLength)) { return false; diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index 2b6b8e2..2834277 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -42,6 +42,19 @@ public readonly BackendMessageCursor Slice(long offset) [MethodImpl(MethodImplOptions.NoInlining)] public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) + { + if (!TryReadNextBuffer(out header, out var fastBuffer, out bufferLength)) + { + buffer = default; + return false; + } + buffer = fastBuffer.Sequence; + return true; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + internal bool TryReadNextBuffer(out BackendHeader header, + out FastReadOnlySequence buffer, out uint bufferLength) { if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) && !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader)) { @@ -67,11 +80,10 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence + internal struct FastReadOnlySequence { object? _startObject; object? _endObject; @@ -143,6 +155,10 @@ public ReadOnlySequence Sequence } } public long Length => _length; + public object? StartObject => _startObject; + public object? EndObject => _endObject; + public int StartIndex => _startIndex; + public int EndIndex => _endIndex; public ReadOnlySpan FirstSpan { From 2c7dbbe529e153d360162d89586dfb03752537f8 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 00:30:13 +0200 Subject: [PATCH 042/136] Let row enumeration classify its publication --- Slon/Pg/CommandResult.cs | 3 ++- .../Flows/CommandFlow.MessageEnumerator.cs | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index fdaa268..8f60e2c 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -415,7 +415,7 @@ Row GetRow() BackendMessage GetCurrentMessage() => _messageEnumerator.Current; BackendMessage.Accessor GetCurrentMessageAccessor() => _messageEnumerator.CurrentAccessor; bool MoveNextMessage() => _messageEnumerator.MoveNext(); - CommandFlow.MoveNextStatus TryMoveNextMessage() => _messageEnumerator.TryMoveNext(); + CommandFlow.MoveNextStatus TryMoveNextMessage() => _messageEnumerator.TryMoveNextRow(); ValueTask MoveNextMessageAsync() => _messageEnumerator.MoveNextAsync(); public struct RowEnumerator : IEnumerator, IAsyncEnumerator @@ -548,6 +548,7 @@ bool HandleUncommon(in BackendMessage.Accessor current) case PgTypes.BackendType.EmptyQueryResponse: case PgTypes.BackendType.CommandComplete: case PgTypes.BackendType.ErrorResponse: + instance._messageEnumerator.MarkCurrentTerminal(); instance.CompleteCommand(current.Message); return false; case PgTypes.BackendType.PortalSuspended when !instance._simpleProtocol: diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 5cbfd21..7fb4f70 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -48,6 +48,9 @@ internal readonly struct ResultMessageEnumerator() : IEnumerator public BackendMessage Current => _messageEnumerator.Current; internal BackendMessage.Accessor CurrentAccessor => _messageEnumerator.CurrentAccessor; internal MoveNextStatus TryMoveNext() => _messageEnumerator.TryMoveNext(); + // RowEnumerator classifies the publication itself, avoiding a second Current copy here. + internal MoveNextStatus TryMoveNextRow() => _messageEnumerator.TryMoveNextRow(); + internal void MarkCurrentTerminal() => _messageEnumerator.MarkCurrentTerminal(); internal ValueTask CollectRowsAsync( TState state, Action collector, CancellationToken cancellationToken) @@ -325,6 +328,25 @@ public MoveNextStatus TryMoveNext() return MoveNextStatus.RequiresInput; } + + public MoveNextStatus TryMoveNextRow() + { + if (_first) + { + _first = false; + return MoveNextStatus.Moved; + } + + _exceptionDispatchInfo?.Throw(); + if (_done) + return MoveNextStatus.EndOfSequence; + + return _decoder.TryMoveNext() + ? MoveNextStatus.Moved + : MoveNextStatus.RequiresInput; + } + + public void MarkCurrentTerminal() => _done = true; public BackendMessage Current { [MethodImpl(MethodImplOptions.AggressiveInlining)] From 6963b504599a498bad2ad44fe4dd60967d65e963 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 00:58:53 +0200 Subject: [PATCH 043/136] Specialize complete prepared message writes --- .../Pg/PgEncoderPreparedExecutionTests.cs | 9 +++--- .../ProtocolDataWriterMessageBudgetTests.cs | 11 +++++++ Slon/Pg/Protocol/PgEncoder.cs | 18 ++++------- Slon/Pg/Protocol/ProtocolDataWriter.cs | 4 +++ Slon/Pg/Protocol/ProtocolWritePipe.cs | 31 ++++++++++++++++--- 5 files changed, 52 insertions(+), 21 deletions(-) diff --git a/Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs b/Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs index 25c4a16..5ec1b5c 100644 --- a/Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs +++ b/Slon.Tests/Pg/PgEncoderPreparedExecutionTests.cs @@ -6,8 +6,8 @@ namespace Slon.Tests.Pg; // The fused prepared-execution write must produce exactly the bytes the message-per-message writers -// produce, for every Describe, Execute, and Sync combination, and must leave the writer's -// per-message framing validation intact. +// produce for every Describe, Execute, and Sync combination, then return the writer at a complete +// message boundary. [TestClass] public class PgEncoderPreparedExecutionTests { @@ -82,13 +82,14 @@ public void UnnamedStatement_WritesEmptyName() } [TestMethod] - public void EveryMessageIsFramedForTheDeclaredLengthCheck() + public void CompleteSequence_LeavesIncrementalTrackerIdle() { var (writer, sink) = NewWriter(); PgEncoder.WritePreparedExecutionCore(writer, Encoding, new EncodedCString("prepared_probe"), describe: true, execute: true, syncCount: 2); - // Arming the next message validates that the previous one was written to its declared length. + Assert.AreEqual(0, writer.CurrentMessagePaddingLength); + // The next incremental message starts from a clean boundary. writer.StartMessage(totalLength: 5); writer.WriteRaw(new byte[5]); writer.Flush(); diff --git a/Slon.Tests/Pg/ProtocolDataWriterMessageBudgetTests.cs b/Slon.Tests/Pg/ProtocolDataWriterMessageBudgetTests.cs index daa2c79..4274617 100644 --- a/Slon.Tests/Pg/ProtocolDataWriterMessageBudgetTests.cs +++ b/Slon.Tests/Pg/ProtocolDataWriterMessageBudgetTests.cs @@ -58,6 +58,17 @@ public void UnderWrite_FaultsAtNextStartMessage() Assert.ThrowsExactly(() => writer.StartMessage(totalLength: 5)); } + [TestMethod] + public void UnderWrite_FaultsBeforeCompleteMessageReservation() + { + var (writer, sink) = NewWriter(); + writer.StartMessage(totalLength: 5); + writer.WriteRaw(new byte[3]); + + Assert.ThrowsExactly(() => writer.GetCompleteMessagesSpan(5)); + Assert.AreEqual(0, sink.ToArray().Length); + } + [TestMethod] public void ExactWrite_Flushes_AllBytesReachWire() { diff --git a/Slon/Pg/Protocol/PgEncoder.cs b/Slon/Pg/Protocol/PgEncoder.cs index 165e87c..fd6f9f9 100644 --- a/Slon/Pg/Protocol/PgEncoder.cs +++ b/Slon/Pg/Protocol/PgEncoder.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Collections.Immutable; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Text; using Slon.Runtime; @@ -250,9 +251,8 @@ internal void WritePreparedExecution(EncodedCString commandName, bool describe, _executionControl.OnMessageWrite(FrontendType.Sync); } - // Each message is still armed and advanced on its own so the per-message declared-length check - // holds. The reserved span stays valid across the advances because the buffering writer only - // reallocates on a reservation it cannot satisfy. + // The complete sequence is sized before its span is acquired and published only after every + // message has been filled. Incremental message tracking remains for streaming writers. internal static void WritePreparedExecutionCore(ProtocolDataWriter writer, Encoding encoding, EncodedCString commandName, bool describe, bool execute, int syncCount) { @@ -268,9 +268,8 @@ internal static void WritePreparedExecutionCore(ProtocolDataWriter writer, Encod + (describe ? header + describeBody : 0) + (execute ? header + executeBody : 0) + syncCount * header); - var span = writer.GetSpan(total); + var span = writer.GetCompleteMessagesSpan(total); - writer.StartMessage(header + bindBody); WriteHeader(span, FrontendType.Bind, bindBody); span[header] = 0; // unnamed portal commandNameBytes.CopyTo(span.Slice(header + 1)); @@ -279,36 +278,31 @@ internal static void WritePreparedExecutionCore(ProtocolDataWriter writer, Encod BinaryPrimitives.WriteUInt16BigEndian(formats.Slice(2), 0); // parameters BinaryPrimitives.WriteUInt16BigEndian(formats.Slice(4), 1); // result format codes BinaryPrimitives.WriteUInt16BigEndian(formats.Slice(6), 1); // all binary - writer.Advance(header + bindBody); span = span.Slice(header + bindBody); if (describe) { - writer.StartMessage(header + describeBody); WriteHeader(span, FrontendType.Describe, describeBody); span[header] = (byte)'P'; span[header + 1] = 0; - writer.Advance(header + describeBody); span = span.Slice(header + describeBody); } if (execute) { - writer.StartMessage(header + executeBody); WriteHeader(span, FrontendType.Execute, executeBody); span[header] = 0; // unnamed portal BinaryPrimitives.WriteUInt32BigEndian(span.Slice(header + 1), 0); // all rows - writer.Advance(header + executeBody); span = span.Slice(header + executeBody); } for (var i = 0; i < syncCount; i++) { - writer.StartMessage(header); WriteHeader(span, FrontendType.Sync, 0); - writer.Advance(header); span = span.Slice(header); } + Debug.Assert(span.IsEmpty); + writer.AdvanceCompleteMessages(total); static void WriteHeader(Span span, FrontendType type, int bodyLength) { diff --git a/Slon/Pg/Protocol/ProtocolDataWriter.cs b/Slon/Pg/Protocol/ProtocolDataWriter.cs index 78bb332..d5d251b 100644 --- a/Slon/Pg/Protocol/ProtocolDataWriter.cs +++ b/Slon/Pg/Protocol/ProtocolDataWriter.cs @@ -90,6 +90,10 @@ internal object GetParameterWriterState(ParameterWriter writer) public Memory GetMemory(int sizeHint = 0) => _pipe.GetMemory(sizeHint); public Span GetSpan(int sizeHint = 0) => _pipe.GetSpan(sizeHint); public void Advance(int count) => _pipe.Advance(count); + internal Span GetCompleteMessagesSpan(int totalLength) + => _pipe.GetCompleteMessagesSpan(totalLength); + internal void AdvanceCompleteMessages(int totalLength) + => _pipe.AdvanceCompleteMessages(totalLength); internal const long UnflushedBytesFlushThreshold = ProtocolWritePipe.UnflushedBytesFlushThreshold; diff --git a/Slon/Pg/Protocol/ProtocolWritePipe.cs b/Slon/Pg/Protocol/ProtocolWritePipe.cs index 1d6515f..153ab2f 100644 --- a/Slon/Pg/Protocol/ProtocolWritePipe.cs +++ b/Slon/Pg/Protocol/ProtocolWritePipe.cs @@ -68,6 +68,25 @@ sealed class ProtocolWritePipe(IOutputWriter writer, Encoding clientEncoding, Ac internal Span GetSpan(int sizeHint = 0) => _bufferingWriter.GetSpan(sizeHint); internal void Advance(int count) => _bufferingWriter.Advance(count); + // A trusted encoder which has computed and filled a complete sequence can publish it once. + // Validate any preceding incrementally-written message before granting the span; until Advance + // nothing from the new sequence is visible to the underlying writer. + internal Span GetCompleteMessagesSpan(int totalLength) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(totalLength); + var unflushed = checked((int)_bufferingWriter.UnflushedBytes); + if (_messageLength is { } previous && unflushed + _messageBytesFlushed != previous) + ThrowUnderwritten(previous, unflushed + _messageBytesFlushed); + return _bufferingWriter.GetSpan(totalLength); + } + + internal void AdvanceCompleteMessages(int totalLength) + { + _bufferingWriter.Advance(totalLength); + _messageLength = null; + _messageBytesFlushed = 0; + } + // Validates the previous message, arms length tracking for the new one, then writes its // five-byte header directly into the buffered span. Keeping these together avoids a second // shell traversal and a temporary header copy. Mid-message flushes are handled by @@ -112,12 +131,13 @@ internal void StartMessage(int totalLength) // re-validates and commits exactly the bytes that remain. void CheckMessageBytesFlushed(int count) { - if (_messageLength is null) + if (!_messageLength.HasValue) return; // Pre-startup raw writes (e.g. StartupMessage's CopyStartupBuffer). + ref readonly var messageLength = ref Nullable.GetValueRefOrDefaultRef(in _messageLength); if (count < 0) throw new ArgumentOutOfRangeException(nameof(count)); - if ((long)_messageBytesFlushed + count > _messageLength) - ThrowOverwritten(_messageLength.Value, _messageBytesFlushed + (long)count); + if ((long)_messageBytesFlushed + count > messageLength) + ThrowOverwritten(messageLength, _messageBytesFlushed + (long)count); } // Commit the counter for bytes that actually left the buffer (the drained delta: before-after @@ -149,10 +169,11 @@ internal int CurrentMessagePaddingLength { get { - if (_messageLength is null) + if (!_messageLength.HasValue) return 0; + ref readonly var messageLength = ref Nullable.GetValueRefOrDefaultRef(in _messageLength); var unflushed = checked((int)_bufferingWriter.UnflushedBytes); - return Math.Max(0, _messageLength.Value - (unflushed + _messageBytesFlushed)); + return Math.Max(0, messageLength - (unflushed + _messageBytesFlushed)); } } From 6cf7bc07611417ab0d1eea05a5cb954d85df002c Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:03:12 +0200 Subject: [PATCH 044/136] Keep exhausted cursor position scalar --- Slon/Pg/Protocol/BackendMessageContext.cs | 8 ++++---- Slon/Pg/Protocol/ProtocolReadPipe.cs | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index a6f10ff..d231357 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -487,18 +487,18 @@ public void RetireCursor(bool retainProjections = false) } public bool TryGetReadRequirement( - out SequencePosition consumed, out long requiredLength) + out long consumedLength, out long requiredLength) { if (!_hasCursor || _cursor.RequiredBufferedLength <= 0) { - consumed = default; + consumedLength = 0; requiredLength = 0; return false; } - consumed = _cursor.UnreadStart; + consumedLength = _cursor.ConsumedLength; requiredLength = _cursor.RequiredBufferedLength - - _cursor.ConsumedLength; + - consumedLength; return true; } diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index df35802..7d71dfd 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -68,12 +68,14 @@ public void PrepareRead() } if (!_messageContext.TryGetReadRequirement( - out var unread, out var requiredLength)) + out var cursorConsumedLength, out var requiredLength)) ThrowHelper.ThrowInvalidOperation( "The current backend-message cursor has not been exhausted."); + var unreadOffset = checked(_pendingCursorOffset + cursorConsumedLength); + var unread = _activeBuffer.GetPosition(unreadOffset); _pendingCursorOffset = retainsResultSet - ? _activeBuffer.Slice(0, unread).Length + ? unreadOffset : 0; _messageContext.RetireCursor( retainProjections: retainsResultSet); From 3eca77d9d7b3c5aca4ed6d215c97dae9b8922e33 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:10:15 +0200 Subject: [PATCH 045/136] Specialize two-type terminal validation --- Slon/Pg/Protocol/BackendMessage.cs | 12 ++++++++++++ Slon/Pg/Protocol/CommandCompleteMessage.cs | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index b8b79c2..83eeb2f 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -429,6 +429,18 @@ static void Throw(BackendType actual, BackendType expected) => throw new PgProtocolException($"Unexpected backend message: {actual}, expected: {expected}."); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void EnsureExpected(BackendType expected1, BackendType expected2) + { + var actual = Type; + if (actual != expected1 && actual != expected2) + Throw(actual, expected1, expected2); + + static void Throw(BackendType actual, BackendType expected1, BackendType expected2) + => throw new PgProtocolException( + $"Unexpected backend message: {actual}, expected: {expected1} or {expected2}."); + } + // Inlining helps as it's usually run over a few RVA items at most. [MethodImpl(MethodImplOptions.AggressiveInlining)] public BackendType EnsureExpected(params ReadOnlySpan expected) diff --git a/Slon/Pg/Protocol/CommandCompleteMessage.cs b/Slon/Pg/Protocol/CommandCompleteMessage.cs index da150a0..9039344 100644 --- a/Slon/Pg/Protocol/CommandCompleteMessage.cs +++ b/Slon/Pg/Protocol/CommandCompleteMessage.cs @@ -56,13 +56,14 @@ or StatementType.Copy or StatementType.Move or StatementType.Fetch or StatementT internal static CommandCompleteMessage Create(in BackendMessage message) { + var header = message.Header; message.EnsureExpected(PgTypes.BackendType.EmptyQueryResponse, PgTypes.BackendType.CommandComplete); message.EnsureBuffered(); - if (message.Header.Type is PgTypes.BackendType.EmptyQueryResponse) + if (header.Type is PgTypes.BackendType.EmptyQueryResponse) return new(StatementType.Empty, 0, 0); Span scratch = stackalloc byte[64]; - var bodyLength = message.Header.BodyLength; + var bodyLength = header.BodyLength; var bytes = message.TryGetFirstSpan(0, out var first) && first.Length >= bodyLength ? first[..bodyLength] : CopyToScratch(message.GetSequence(), scratch); From cb950f682b916be5fe8a70621c9e34f09b26b091 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:18:04 +0200 Subject: [PATCH 046/136] Skip fragmented parsing below one header --- Slon/Pg/Protocol/BackendMessageCursor.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index 2834277..b383496 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -56,7 +56,9 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) { - if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) && !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader)) + if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) + && (_buffer.Length < Header.ByteCount + || !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader))) { _requiredBufferedLength = ConsumedLength + Header.ByteCount; buffer = default; From 191d0e02b79c0f50f93a8cf38f61a9c4590bf366 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:23:04 +0200 Subject: [PATCH 047/136] Complete flows from their structural owner --- Slon/Pg/Protocol/PgClientFlow.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 9f64653..6de98d5 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -169,7 +169,6 @@ void IThreadPoolWorkItem.Execute() // pattern). At most one pending waiter per tenure; post-completion awaits resolve // synchronously. Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _completionCore; - int _completionClaim; ManualResetEventSlim? _completionEvent; // 1 while a WaitForComplete token is live (set at capture, cleared after GetResult consumed the // core). Guards reuse: Reset bumps the core's version, so it must not run while this is set. @@ -298,8 +297,10 @@ void ResetActivationSource() void CompleteFlow(Exception? exception) { - if (Interlocked.CompareExchange(ref _completionClaim, 1, 0) != 0) - return; + // Completion has one structural owner: either pipeline retirement, or the source drain for + // an item which was never dispatched. Substitution transfers failed-item completion to the + // policy; it does not add a competing completer. + Debug.Assert(_completionCore.GetStatus(_completionCore.Version) is ValueTaskSourceStatus.Pending); if (exception is null) _completionCore.SetResult(default, runContinuationsAsynchronously: true); else @@ -408,7 +409,6 @@ public void Reset() // Version bump per tenure. Cross-tenure completer staleness rests on the done -> torn-down // -> retired layering (Complete precedes recycle), the same basis as the rest of this reset. _completionCore.Reset(); - Volatile.Write(ref _completionClaim, 0); _completionEvent?.Reset(); ResetActivationSource(); _rfqCount = 0; From abdf6e3580c39c109f4c43b3dc0138b6d22596a7 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:46:11 +0200 Subject: [PATCH 048/136] Assign reference-bearing values granularly --- Slon/Pg/CommandDescriptor.cs | 9 +++++++++ Slon/Pg/CommandResult.cs | 2 +- Slon/Pg/ParameterTypeList.cs | 10 ++++++++++ Slon/Pg/Protocol/BackendMessage.cs | 12 ++++-------- Slon/Pg/Row.cs | 2 +- Slon/Text/EncodedCString.cs | 6 ++++++ 6 files changed, 31 insertions(+), 10 deletions(-) diff --git a/Slon/Pg/CommandDescriptor.cs b/Slon/Pg/CommandDescriptor.cs index d7a22af..c4dc02c 100644 --- a/Slon/Pg/CommandDescriptor.cs +++ b/Slon/Pg/CommandDescriptor.cs @@ -76,4 +76,13 @@ public static CommandDescriptor CreatePrepared(EncodedCString commandName, Param public static CommandDescriptor Create(string commandText, ParameterTypeList parameterTypes = default, EncodedCString commandName = default) => new(commandText, parameterTypes, commandName); + + // Fieldwise equivalent of assignment, avoiding stores for reference components already present. + internal static void Assign(ref CommandDescriptor destination, in CommandDescriptor value) + { + if (!ReferenceEquals(destination._rowDescriptionOrCommandText, value._rowDescriptionOrCommandText)) + Unsafe.AsRef(in destination._rowDescriptionOrCommandText) = value._rowDescriptionOrCommandText; + EncodedCString.Assign(ref Unsafe.AsRef(in destination._commandName), in value._commandName); + ParameterTypeList.Assign(ref Unsafe.AsRef(in destination._parameterTypes), in value._parameterTypes); + } } diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index 8f60e2c..53b84f7 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -45,7 +45,7 @@ internal void Initialize(PgClientFlow flow, int index, CommandDescriptor descrip if (!ReferenceEquals(_flow, flow)) _flow = flow; _index = index; - _descriptor = descriptor; + CommandDescriptor.Assign(ref _descriptor, in descriptor); // If the command wasn't redescribed, and the prepared description is valid use it instead. var rowDescription = requestedRowDescription; diff --git a/Slon/Pg/ParameterTypeList.cs b/Slon/Pg/ParameterTypeList.cs index 0c8af02..19748d2 100644 --- a/Slon/Pg/ParameterTypeList.cs +++ b/Slon/Pg/ParameterTypeList.cs @@ -2,6 +2,7 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Slon.Pg.Types; @@ -167,4 +168,13 @@ public bool Equals(ParameterTypeList other) public override int GetHashCode() => HashCode.Combine(_source, _writer, _count); public static bool operator ==(ParameterTypeList left, ParameterTypeList right) => left.Equals(right); public static bool operator !=(ParameterTypeList left, ParameterTypeList right) => !left.Equals(right); + + internal static void Assign(ref ParameterTypeList destination, in ParameterTypeList value) + { + if (!ReferenceEquals(destination._source, value._source)) + Unsafe.AsRef(in destination._source) = value._source; + if (!ReferenceEquals(destination._writer, value._writer)) + Unsafe.AsRef(in destination._writer) = value._writer; + Unsafe.AsRef(in destination._count) = value._count; + } } diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index 83eeb2f..2fae545 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -59,18 +59,14 @@ internal static void InitializeIndependent(ref BackendMessage destination, BackendHeader header, ReadOnlySequence buffer) { var value = CreateIndependent(header, buffer); - WriteGranularly(ref destination, in value); + Assign(ref destination, in value); } - internal static void Copy( - ref BackendMessage destination, in BackendMessage value) - => WriteGranularly(ref destination, in value); - internal static void Initialize(ref BackendMessage destination, BackendHeader header, ReadOnlySequence buffer, BackendMessageContext context, short token, bool buffered) { var value = new BackendMessage(header, buffer, context, token, buffered); - WriteGranularly(ref destination, in value, destinationIsZero: false); + Assign(ref destination, in value, destinationIsZero: false); } internal static void Initialize(ref BackendMessage destination, BackendHeader header, @@ -95,7 +91,7 @@ internal static void Initialize(ref BackendMessage destination, BackendHeader he // The JIT should have a phase for picking granular writes (and write barriers) over full struct assignments. // This translation is entirely mechanical (even though these implementations need to deviate for external types). [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void WriteGranularly(ref BackendMessage destination, in BackendMessage value, bool destinationIsZero = false) + static void Assign(ref BackendMessage destination, in BackendMessage value, bool destinationIsZero = false) { if ((destinationIsZero && value._contextOrEndObject is not null) || !ReferenceEquals(destination._contextOrEndObject, value._contextOrEndObject)) @@ -379,7 +375,7 @@ internal ValueTask BufferBodyAsync(CancellationToken cancellationToken) // The JIT should have a phase for picking granular writes (and write barriers) over full struct assignments. // This translation is entirely mechanical (even though these implementations need to deviate for external types). - internal static void WriteGranularly(ref Accessor destination, in Accessor value, bool destinationIsZero = false) + internal static void Assign(ref Accessor destination, in Accessor value, bool destinationIsZero = false) { if ((destinationIsZero && value._context is not null) || !ReferenceEquals(destination._context, value._context)) Unsafe.AsRef(in destination._context) = value._context!; diff --git a/Slon/Pg/Row.cs b/Slon/Pg/Row.cs index d18b5cf..10448fa 100644 --- a/Slon/Pg/Row.cs +++ b/Slon/Pg/Row.cs @@ -619,7 +619,7 @@ internal void InitializeRow(in BackendMessage.Accessor row) _column = 0; _columnOffset = sizeof(short); _lastBufferedOrdinal = -1; - BackendMessage.Accessor.WriteGranularly(ref _messageAccessor, row); + BackendMessage.Accessor.Assign(ref _messageAccessor, row); CaptureBufferedBody(row); } diff --git a/Slon/Text/EncodedCString.cs b/Slon/Text/EncodedCString.cs index 1260feb..009248f 100644 --- a/Slon/Text/EncodedCString.cs +++ b/Slon/Text/EncodedCString.cs @@ -35,6 +35,12 @@ public bool ValueEquals(EncodedCString other) public static implicit operator EncodedCString(string value) => new(value); + internal static void Assign(ref EncodedCString destination, in EncodedCString value) + { + if (!ReferenceEquals(destination._core, value._core)) + Unsafe.AsRef(in destination._core) = value._core; + } + // Used for long lived strings that may have to be re-encoded (but usually wont), thread-safe. [DebuggerDisplay("{_value,nq}")] sealed class Core(string value) From 0be6ce66edb1b71cde066d72fa411c017eed5cc6 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:53:01 +0200 Subject: [PATCH 049/136] Read transaction status directly from buffered messages --- Slon/Pg/Protocol/BackendMessage.cs | 25 ++++++++++++++++++++++++ Slon/Pg/Protocol/ReadyForQueryMessage.cs | 8 +------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessage.cs b/Slon/Pg/Protocol/BackendMessage.cs index 2fae545..f679c79 100644 --- a/Slon/Pg/Protocol/BackendMessage.cs +++ b/Slon/Pg/Protocol/BackendMessage.cs @@ -214,6 +214,31 @@ internal bool TryGetFirstSpanUnchecked(int offset, out ReadOnlySpan span) return false; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool TryGetFirstByte(int offset, out byte value) + { + Debug.Assert(Buffered); + offset += BackendHeader.ByteCount; + var firstLength = IsIndependent + ? _endIndexOrBufferedLength - _startIndex + : _endIndexOrBufferedLength; + if (_firstObject is byte[] array && (uint)offset < (uint)firstLength) + { + value = array[_startIndex + offset]; + return true; + } + + var firstMemory = GetFirstMemory(); + if ((uint)offset < (uint)firstMemory.Length) + { + value = firstMemory.Span[offset]; + return true; + } + + value = 0; + return false; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool TryGetBufferedFirstMemory(int offset, out ReadOnlyMemory memory) { diff --git a/Slon/Pg/Protocol/ReadyForQueryMessage.cs b/Slon/Pg/Protocol/ReadyForQueryMessage.cs index 977e4ce..8a4703c 100644 --- a/Slon/Pg/Protocol/ReadyForQueryMessage.cs +++ b/Slon/Pg/Protocol/ReadyForQueryMessage.cs @@ -15,14 +15,8 @@ public static ReadyForQueryMessage Create(in BackendMessage message) message.EnsureExpected(PgTypes.BackendType.ReadyForQuery); message.EnsureBuffered(); - byte status; - if (message.TryGetFirstSpan(0, out var body) && !body.IsEmpty) + if (!message.TryGetFirstByte(0, out var status)) { - status = body[0]; - } - else - { - status = 0; message.BodyReader.TryCopyTo(new Span(ref status)); } var transactionStatus = (TransactionStatus)status; From a8d87d0663569f73b89217d1c0777e12e076c243 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:58:30 +0200 Subject: [PATCH 050/136] Reuse direct decoder read state machine --- Slon/Pg/Protocol/PgDecoder.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 2d288fa..90c936b 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -570,6 +570,7 @@ ValueTask AwaitDirectRead( : MoveNextDirectAsync(directReadTask, frontierFlow); [MethodImpl(MethodImplOptions.NoInlining)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask MoveNextDirectAsync( ValueTask directReadTask, PgClientFlow frontierFlow) From 27b701a28747103276cd34906c5dbb23647a33e4 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 02:19:34 +0200 Subject: [PATCH 051/136] Reuse streamed message state machines --- Slon/Pg/Protocol/BackendMessageBodyReader.cs | 3 +++ Slon/Pg/Protocol/BackendMessageContext.cs | 1 + Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs | 1 + Slon/Pg/Protocol/PgDecoder.cs | 1 + Slon/Pg/Row.cs | 1 + 5 files changed, 7 insertions(+) diff --git a/Slon/Pg/Protocol/BackendMessageBodyReader.cs b/Slon/Pg/Protocol/BackendMessageBodyReader.cs index 1ecbc15..f93814b 100644 --- a/Slon/Pg/Protocol/BackendMessageBodyReader.cs +++ b/Slon/Pg/Protocol/BackendMessageBodyReader.cs @@ -74,6 +74,7 @@ public ValueTask ReadAsync(CancellationToken cancellationToken = default) } return Core(task); + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(ValueTask task) => Publish(await task.ConfigureAwait(false)); } @@ -104,6 +105,7 @@ public ValueTask ExtendAsync(CancellationToken cancellationToken = default) } return Core(task); + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(ValueTask task) => Publish(await task.ConfigureAwait(false), retained: true); } @@ -173,6 +175,7 @@ public ValueTask BufferAllAsync(CancellationToken cancellationToken = default) return default; return Core(cancellationToken); + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(CancellationToken cancellationToken) { while (!IsComplete) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index d231357..dd7ee24 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -327,6 +327,7 @@ public bool TryExtend(short token, out CurrentMessageBuffer result) return true; } + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] public async ValueTask ExtendAsync(short token, CancellationToken cancellationToken) { EnsureBodyWindowAvailable(token); diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 7fb4f70..2165df0 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -149,6 +149,7 @@ public ValueTask MoveNextAsync() return Core(); + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask Core() { try diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 90c936b..7075954 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -320,6 +320,7 @@ internal ValueTask ExtendCurrentMessageAsync(CancellationT return Core(cancellationToken); + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask Core(CancellationToken cancellationToken) { var timeoutSet = false; diff --git a/Slon/Pg/Row.cs b/Slon/Pg/Row.cs index 10448fa..d34903e 100644 --- a/Slon/Pg/Row.cs +++ b/Slon/Pg/Row.cs @@ -532,6 +532,7 @@ internal ValueTask BufferAllAsync(CancellationToken cancellationToken = default) return default; return Core(cancellationToken); + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(CancellationToken token) { await _bodyReader.BufferAllAsync(token).ConfigureAwait(false); From bea9508862fb0ef5c6a8a4d716cdcc84c432c970 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 02:34:37 +0200 Subject: [PATCH 052/136] Bypass policy state machine for settled execution --- Slon/Pg/Protocol/PgClientProtocol.cs | 45 +++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index eb3592b..e11b029 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1334,22 +1334,53 @@ public ValueTask ExecuteItemAsync(PgClientFlow item, bool pi PromiseAsyncValueTaskMethodBuilder.Promise = null; } - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] - static async ValueTask ExecuteCore( + static ValueTask ExecuteCore( Control control, PgClientFlow item, CancellationToken cancellationToken) { - await control.WaitForCancellationAttempt().ConfigureAwait(false); + var pending = control.WaitForCancellationAttempt(); + if (!pending.IsCompletedSuccessfully) + return AwaitPriorWork(control, item, cancellationToken, pending, checkFlush: true); + pending.GetAwaiter().GetResult(); // A flow may defer this flush only when its first phase cannot wait for decoder input. if (!item.SupportsDeferredFlush && control.UnflushedBytes != 0) - await control.FlushAsync(cancellationToken).ConfigureAwait(false); + { + pending = control.FlushAsync(cancellationToken); + if (!pending.IsCompletedSuccessfully) + return AwaitPriorWork(control, item, cancellationToken, pending, checkFlush: false); + pending.GetAwaiter().GetResult(); + } + + var execution = control.Execute(item); + if (!execution.IsCompletedSuccessfully) + return AwaitExecution(execution); + var tasks = execution.GetAwaiter().GetResult(); + return new ValueTask( + new PipelineItemResult(tasks.TrailingExecutionTask, tasks.PipelineTask)); + } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] + static async ValueTask AwaitPriorWork( + Control control, PgClientFlow item, CancellationToken cancellationToken, + ValueTask pending, bool checkFlush) + { + await pending.ConfigureAwait(false); + if (checkFlush && !item.SupportsDeferredFlush && control.UnflushedBytes != 0) + await control.FlushAsync(cancellationToken).ConfigureAwait(false); var tasks = await control.Execute(item).ConfigureAwait(false); - return new PipelineItemResult(tasks.TrailingExecutionTask, tasks.PipelineTask); + return new(tasks.TrailingExecutionTask, tasks.PipelineTask); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] + static async ValueTask AwaitExecution(ValueTask execution) + { + var tasks = await execution.ConfigureAwait(false); + return new(tasks.TrailingExecutionTask, tasks.PipelineTask); } - // Stock builder (no shared promise) for pipeline-task recovery. Body identical to ExecuteCore. + // Stock builder (no shared promise) for pipeline-task recovery. static async ValueTask ExecutePipelineTaskRecovery( Control control, PgClientFlow item, CancellationToken cancellationToken) { From 66491ed8dfbe20f1a8b597d65f049beabdd9eb76 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 03:14:58 +0200 Subject: [PATCH 053/136] Fold message projection into its reader continuation --- Slon/Pg/Protocol/BackendMessageBodyReader.cs | 7 ++++--- Slon/Pg/Protocol/BackendMessageContext.cs | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageBodyReader.cs b/Slon/Pg/Protocol/BackendMessageBodyReader.cs index f93814b..be19969 100644 --- a/Slon/Pg/Protocol/BackendMessageBodyReader.cs +++ b/Slon/Pg/Protocol/BackendMessageBodyReader.cs @@ -97,17 +97,18 @@ public bool TryExtend() public ValueTask ExtendAsync(CancellationToken cancellationToken = default) { EnsureCanExtend(); - var task = _context.ExtendAsync(_token, cancellationToken); + var task = _context.BeginExtendAsync(_token, cancellationToken); if (task.IsCompletedSuccessfully) { - Publish(task.Result, retained: true); + Publish(_context.CompleteExtend(_token, task.Result), retained: true); return default; } return Core(task); [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(ValueTask task) - => Publish(await task.ConfigureAwait(false), retained: true); + => Publish(_context.CompleteExtend( + _token, await task.ConfigureAwait(false)), retained: true); } public void Extend() diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index dd7ee24..8d58d00 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -327,13 +327,15 @@ public bool TryExtend(short token, out CurrentMessageBuffer result) return true; } - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - public async ValueTask ExtendAsync(short token, CancellationToken cancellationToken) + public ValueTask BeginExtendAsync(short token, CancellationToken cancellationToken) { EnsureBodyWindowAvailable(token); - return GetBodyBuffer(token, await _decoder.ExtendCurrentMessageAsync(cancellationToken).ConfigureAwait(false)); + return _decoder.ExtendCurrentMessageAsync(cancellationToken); } + public CurrentMessageBuffer CompleteExtend(short token, CurrentMessageBuffer result) + => GetBodyBuffer(token, result); + public CurrentMessageBuffer Extend(short token) { EnsureBodyWindowAvailable(token); @@ -391,7 +393,11 @@ public ValueTask BufferCurrentMessageAsync(short token, CancellationToken cancel async ValueTask Core(short token, CancellationToken cancellationToken) { CurrentMessageBuffer result; - do result = await ExtendAsync(token, cancellationToken).ConfigureAwait(false); + do + { + result = CompleteExtend(token, + await BeginExtendAsync(token, cancellationToken).ConfigureAwait(false)); + } while (!result.IsComplete); } } From b2a0529c48c5adbb25e9d1368df0014df8ecaf9d Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 03:14:59 +0200 Subject: [PATCH 054/136] Fold pipe completion into the decoder continuation --- Slon/Pg/Protocol/PgDecoder.cs | 6 ++++-- Slon/Pg/Protocol/ProtocolReadPipe.cs | 25 +++++-------------------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 7075954..2befe9a 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -262,8 +262,9 @@ async ValueTask Core(CancellationToken cancellationToken) { ArmReadTimeout(); timeoutSet = true; - return await _pipe.SlideCurrentMessageAsync( + var read = await _pipe.BeginSlideCurrentMessageAsync( consumed, consumedLength, _cancellationTokenSource.Token).ConfigureAwait(false); + return _pipe.CompleteCurrentMessageRead(read, _cancellationTokenSource.Token); } catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) { @@ -331,8 +332,9 @@ async ValueTask Core(CancellationToken cancellationToken) { ArmReadTimeout(); timeoutSet = true; - return await _pipe.ExtendCurrentMessageAsync( + var read = await _pipe.BeginExtendCurrentMessageAsync( _cancellationTokenSource.Token).ConfigureAwait(false); + return _pipe.CompleteCurrentMessageRead(read, _cancellationTokenSource.Token); } catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) { diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index 7d71dfd..8d6bd57 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -1,6 +1,5 @@ using System.Buffers; using System.IO.Pipelines; -using System.Runtime.CompilerServices; using Slon.Pipelines; namespace Slon.Pg.Protocol; @@ -228,13 +227,12 @@ public bool TrySlideCurrentMessage( return true; } - public ValueTask SlideCurrentMessageAsync( + public ValueTask BeginSlideCurrentMessageAsync( SequencePosition consumed, long consumedLength, CancellationToken cancellationToken) { PrepareCurrentMessageRead(consumed, consumedLength, PendingRead.Slide); - return CompleteCurrentMessageReadAsync( - reader.ReadAsync(cancellationToken), cancellationToken); + return reader.ReadAsync(cancellationToken); } public CurrentMessageBuffer SlideCurrentMessage( @@ -259,13 +257,12 @@ public bool TryExtendCurrentMessage(out CurrentMessageBuffer result) return true; } - public ValueTask ExtendCurrentMessageAsync( + public ValueTask BeginExtendCurrentMessageAsync( CancellationToken cancellationToken) { PrepareCurrentMessageRead( _retainedStart, consumedLength: 0, PendingRead.Extend); - return CompleteCurrentMessageReadAsync( - reader.ReadAsync(cancellationToken), cancellationToken); + return reader.ReadAsync(cancellationToken); } public CurrentMessageBuffer ExtendCurrentMessage(TimeSpan timeout) @@ -314,19 +311,7 @@ mode is PendingRead.Slide && !_retainsResultSet _pendingRead = mode; } - ValueTask CompleteCurrentMessageReadAsync( - ValueTask task, CancellationToken cancellationToken) - => task.IsCompletedSuccessfully - ? new(CompleteCurrentMessageRead(task.Result, cancellationToken)) - : Core(task, cancellationToken); - - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask Core( - ValueTask task, CancellationToken cancellationToken) - => CompleteCurrentMessageRead( - await task.ConfigureAwait(false), cancellationToken); - - CurrentMessageBuffer CompleteCurrentMessageRead( + public CurrentMessageBuffer CompleteCurrentMessageRead( in ReadResult result, CancellationToken cancellationToken = default) { if (_pendingRead is not (PendingRead.Slide or PendingRead.Extend)) From 0ec71893ed7bf923702a01527f070bcf7eca6249 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 03:21:18 +0200 Subject: [PATCH 055/136] Read buffered message bodies to their boundary --- Slon/Pg/Protocol/BackendMessageBodyReader.cs | 9 ++- Slon/Pg/Protocol/BackendMessageContext.cs | 6 ++ Slon/Pg/Protocol/PgDecoder.cs | 71 +++++++++++--------- Slon/Pg/Protocol/ProtocolReadPipe.cs | 11 +++ 4 files changed, 66 insertions(+), 31 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageBodyReader.cs b/Slon/Pg/Protocol/BackendMessageBodyReader.cs index be19969..1acaf70 100644 --- a/Slon/Pg/Protocol/BackendMessageBodyReader.cs +++ b/Slon/Pg/Protocol/BackendMessageBodyReader.cs @@ -180,7 +180,14 @@ public ValueTask BufferAllAsync(CancellationToken cancellationToken = default) async ValueTask Core(CancellationToken cancellationToken) { while (!IsComplete) - await ExtendAsync(cancellationToken).ConfigureAwait(false); + { + EnsureCanExtend(); + var task = _context.BeginBufferAsync(_token, cancellationToken); + var result = task.IsCompletedSuccessfully + ? task.Result + : await task.ConfigureAwait(false); + Publish(_context.CompleteExtend(_token, result), retained: true); + } } } diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 8d58d00..0c9130a 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -333,6 +333,12 @@ public ValueTask BeginExtendAsync(short token, Cancellatio return _decoder.ExtendCurrentMessageAsync(cancellationToken); } + public ValueTask BeginBufferAsync(short token, CancellationToken cancellationToken) + { + EnsureBodyWindowAvailable(token); + return _decoder.BufferCurrentMessageAsync(cancellationToken); + } + public CurrentMessageBuffer CompleteExtend(short token, CurrentMessageBuffer result) => GetBodyBuffer(token, result); diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 2befe9a..37daeb1 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -319,38 +319,49 @@ internal ValueTask ExtendCurrentMessageAsync(CancellationT if (_pipe.TryExtendCurrentMessage(out var result)) return new(result); - return Core(cancellationToken); + return ReadCurrentMessageAsync(cancellationToken, bufferAll: false); + } - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask Core(CancellationToken cancellationToken) + internal ValueTask BufferCurrentMessageAsync( + CancellationToken cancellationToken) + { + EnsureUsableCts(); + return ReadCurrentMessageAsync(cancellationToken, bufferAll: true); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask ReadCurrentMessageAsync( + CancellationToken cancellationToken, bool bufferAll) + { + var timeoutSet = false; + var frontierFlow = EnterCancellationReadFrontier(); + var registration = cancellationToken.UnsafeRegister( + static (state, _) => ((CancellationTokenSource)state!).Cancel(), _cancellationTokenSource); + try { - var timeoutSet = false; - var frontierFlow = EnterCancellationReadFrontier(); - var registration = cancellationToken.UnsafeRegister( - static (state, _) => ((CancellationTokenSource)state!).Cancel(), _cancellationTokenSource); - try - { - ArmReadTimeout(); - timeoutSet = true; - var read = await _pipe.BeginExtendCurrentMessageAsync( - _cancellationTokenSource.Token).ConfigureAwait(false); - return _pipe.CompleteCurrentMessageRead(read, _cancellationTokenSource.Token); - } - catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) - { - throw TranslateReadCancellation(ex, cancellationToken); - } - catch (EndOfStreamException ex) - { - throw TranslateEof(ex); - } - finally - { - LeaveCancellationReadFrontier(frontierFlow); - registration.Dispose(); - if (timeoutSet) - SetRemainingTimeout(Timeout.InfiniteTimeSpan); - } + ArmReadTimeout(); + timeoutSet = true; + var read = await (bufferAll + ? _pipe.BeginBufferCurrentMessageAsync(_cancellationTokenSource.Token) + : _pipe.BeginExtendCurrentMessageAsync(_cancellationTokenSource.Token)) + .ConfigureAwait(false); + return _pipe.CompleteCurrentMessageRead(read, _cancellationTokenSource.Token); + } + catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) + { + throw TranslateReadCancellation(ex, cancellationToken); + } + catch (EndOfStreamException ex) + { + throw TranslateEof(ex); + } + finally + { + LeaveCancellationReadFrontier(frontierFlow); + registration.Dispose(); + if (timeoutSet) + SetRemainingTimeout(Timeout.InfiniteTimeSpan); } } diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index 8d6bd57..c12dd32 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -265,6 +265,17 @@ public ValueTask BeginExtendCurrentMessageAsync( return reader.ReadAsync(cancellationToken); } + public ValueTask BeginBufferCurrentMessageAsync( + CancellationToken cancellationToken) + { + PrepareCurrentMessageRead( + _retainedStart, consumedLength: 0, PendingRead.Extend); + var requiredLength = checked(_currentMessageOffset + _currentMessageLength); + return requiredLength <= int.MaxValue + ? reader.ReadAtLeastAsync((int)requiredLength, cancellationToken) + : reader.ReadAsync(cancellationToken); + } + public CurrentMessageBuffer ExtendCurrentMessage(TimeSpan timeout) { if (reader is not StreamPipeReader syncReader) From 0b499f33364703a293013faf620d5571e5d90953 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 03:30:15 +0200 Subject: [PATCH 056/136] Reuse parsed backend message length --- Slon/Pg/Protocol/BackendMessageCursor.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index b383496..d975ec9 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -68,11 +68,12 @@ internal bool TryReadNextBuffer(out BackendHeader header, } var backendType = (BackendType)protoHeader.Tag; - if (protoHeader.MessageLength > MaxMessageLength) - ThrowMessageTooLong(protoHeader.MessageLength); + var messageLength = protoHeader.MessageLength; + if (messageLength > MaxMessageLength) + ThrowMessageTooLong(messageLength); var required = backendType is BackendType.DataRow - ? Math.Min(protoHeader.MessageLength, (uint)_dataRowStreamingThreshold) - : protoHeader.MessageLength; + ? Math.Min(messageLength, (uint)_dataRowStreamingThreshold) + : messageLength; if (_buffer.Length < required) { _requiredBufferedLength = ConsumedLength + required; @@ -82,7 +83,7 @@ internal bool TryReadNextBuffer(out BackendHeader header, return false; } - buffer = _buffer.SplitInPlace(Math.Min(_buffer.Length, protoHeader.MessageLength)); + buffer = _buffer.SplitInPlace(Math.Min(_buffer.Length, messageLength)); _requiredBufferedLength = 0; Debug.Assert(buffer.Length <= uint.MaxValue); bufferLength = unchecked((uint)buffer.Length); From fe42cf91c8731cffcb35b9d0a9b5fc838fac9067 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 03:32:58 +0200 Subject: [PATCH 057/136] Reuse backend publication length check --- Slon/Pg/Protocol/BackendMessageContext.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 0c9130a..1e50db1 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -544,13 +544,15 @@ public bool TryPeekNext(out BackendHeader header) { return false; } - if (bufferLength < header.MessageLength) + var messageLength = header.MessageLength; + var buffered = bufferLength >= messageLength; + if (!buffered) _decoder.SetCurrentMessageLength( _cursor.ConsumedLength - bufferLength - + header.MessageLength); + + messageLength); _messageState = 0; BackendMessage.Initialize(ref _current, header, buffer, this, ++_version, - bufferLength >= header.MessageLength); + buffered); _publicationState = PublicationState.Peeked; return true; } From 04758effc23eb14c5ee913864f0aa69757ec53e4 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 04:02:33 +0200 Subject: [PATCH 058/136] Keep optional flow lifecycle state cold --- Slon/Pg/Protocol/PgClientFlow.cs | 48 ++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 6de98d5..7158616 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -79,6 +79,12 @@ internal void ResetInteraction() [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] public abstract class PgClientFlow : IValueTaskSource, IValueTaskSource, IThreadPoolWorkItem { + sealed class OptionalState + { + internal ManualResetEventSlim? CompletionEvent; + internal CancellationTokenRegistration ActivationRegistration; + } + PgClientProtocol.Control? _pendingActivationControl; FlowEnqueueOptions _enqueueOptions; bool _ownsWireCapacity; @@ -169,7 +175,7 @@ void IThreadPoolWorkItem.Execute() // pattern). At most one pending waiter per tenure; post-completion awaits resolve // synchronously. Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _completionCore; - ManualResetEventSlim? _completionEvent; + OptionalState? _optionalState; // 1 while a WaitForComplete token is live (set at capture, cleared after GetResult consumed the // core). Guards reuse: Reset bumps the core's version, so it must not run while this is set. int _completionWaiterPending; @@ -177,7 +183,6 @@ void IThreadPoolWorkItem.Execute() // Activation state. Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _activationTaskSource; int _activationClaim; - CancellationTokenRegistration _activationCancellationTokenRegistration; TimeSpan _remainingActivationTimeout; bool _pendingTimeoutStarted; @@ -273,6 +278,21 @@ protected PgClientFlow(bool supportsDeferredFlush = false) _supportsDeferredFlush = supportsDeferredFlush; } + OptionalState GetOrCreateOptionalState() + => Volatile.Read(ref _optionalState) + ?? Interlocked.CompareExchange(ref _optionalState, new(), null) + ?? _optionalState; + + CancellationTokenRegistration TakeActivationRegistration() + { + var state = Volatile.Read(ref _optionalState); + if (state is null) + return default; + var registration = state.ActivationRegistration; + state.ActivationRegistration = default; + return registration; + } + bool TrySetActivationResult(bool runContinuationsAsynchronously) { if (Interlocked.CompareExchange(ref _activationClaim, 1, 0) != 0) @@ -366,11 +386,13 @@ internal void WaitForCompleteSynchronously(CancellationToken cancellationToken = cancellationToken.ThrowIfCancellationRequested(); Volatile.Write(ref _completionWaiterPending, 1); var token = _completionCore.Version; - var completionEvent = _completionEvent; + var state = GetOrCreateOptionalState(); + var completionEvent = state.CompletionEvent; if (completionEvent is null) { var created = new ManualResetEventSlim(); - completionEvent = Interlocked.CompareExchange(ref _completionEvent, created, null) ?? created; + completionEvent = Interlocked.CompareExchange( + ref state.CompletionEvent, created, null) ?? created; if (!ReferenceEquals(completionEvent, created)) created.Dispose(); } @@ -409,7 +431,7 @@ public void Reset() // Version bump per tenure. Cross-tenure completer staleness rests on the done -> torn-down // -> retired layering (Complete precedes recycle), the same basis as the rest of this reset. _completionCore.Reset(); - _completionEvent?.Reset(); + Volatile.Read(ref _optionalState)?.CompletionEvent?.Reset(); ResetActivationSource(); _rfqCount = 0; _cancellationWindow = 0; @@ -894,12 +916,13 @@ public ValueTask ExecuteAuto() public void Activate() { - flow._activationCancellationTokenRegistration.Dispose(); + var activationRegistration = flow.TakeActivationRegistration(); + activationRegistration.Dispose(); // If none of the cancellations triggered, we have a problem, throw. if (!flow.TrySetActivationResult(runContinuationsAsynchronously: false) && !(flow._remainingActivationTimeout <= TimeSpan.Zero) && !control.AbortToken.IsCancellationRequested - && !flow._activationCancellationTokenRegistration.Token.IsCancellationRequested) + && !activationRegistration.Token.IsCancellationRequested) ThrowHelper.ThrowInvalidOperation("Flow was already activated unexpectedly."); } @@ -986,7 +1009,7 @@ public void DetachForMigration(PgClientFlowSource source) Debug.Assert(!flow._started && !flow._completed); // No body has awaited activation yet, so there is no activation-cycle registration to // dismantle and reconstruct on the replacement source. - Debug.Assert(flow._activationCancellationTokenRegistration == default); + Debug.Assert(Volatile.Read(ref flow._optionalState)?.ActivationRegistration == default); // Forceful shutdown may have propagated the retired wire's abort into this inert flow // before the source drain transferred it. No body can have observed the pre-start gate; // reset that wire-local verdict so replacement dispatch can activate the same operation. @@ -1017,7 +1040,7 @@ public void Release(Exception? exception = null) flow._completed = true; if (flow.OwnsAdmissionBarrier) control.ReleaseAdmissionBarrier(); - flow._activationCancellationTokenRegistration.Dispose(); + flow.TakeActivationRegistration().Dispose(); var observer = flow._observer; var observerState = flow._observerState; try { flow.OnReleasing(exception); } @@ -1042,7 +1065,7 @@ public void Release(Exception? exception = null) // inline caller continuations are a re-entrancy hazard, the contract the old TCS's // RunContinuationsAsynchronously carried, minus its unconditional thread-pool destination. flow.CompleteFlow(exception); - flow._completionEvent?.Set(); + Volatile.Read(ref flow._optionalState)?.CompletionEvent?.Set(); // The completed observer runs from CompleteItem in the advancer/retirement work-item // context: a raw throw would crash that thread unobserved. Don't swallow either - a // throwing completed observer means the consumer-side integration is broken, so the @@ -1117,9 +1140,10 @@ public void RegisterActivationCancellation(CancellationToken cancellationToken) { if (!cancellationToken.CanBeCanceled) return; - if (flow._activationCancellationTokenRegistration != default) + var state = flow.GetOrCreateOptionalState(); + if (state.ActivationRegistration != default) ThrowHelper.ThrowInvalidOperation("Concurrent activation result awaits are not supported."); - flow._activationCancellationTokenRegistration = cancellationToken.UnsafeRegister( + state.ActivationRegistration = cancellationToken.UnsafeRegister( static (state, token) => ((PgClientFlow)state!).TrySetActivationException(new OperationCanceledException(token), runContinuationsAsynchronously: true), flow); From d7ec45d5f2d0dbc81a476c91dcbd085d24ec2a38 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 04:09:39 +0200 Subject: [PATCH 059/136] Keep command execution promise across idle reads --- Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs | 7 +++++-- Slon/Pg/Protocol/Flows/CommandFlow.cs | 3 ++- Slon/Pg/Protocol/PgClientProtocol.cs | 7 ++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 2165df0..3a177bc 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -19,14 +19,12 @@ internal struct ReadState { public ResultMessageEnumerator ResultMessageEnumerator { get; } public CommandResult CommandResult { get; } - public ValueTaskSourcePromise ReadPromise { get; } public RowDescription RowDescription { get; } public ReadState() { ResultMessageEnumerator = new(); CommandResult = new(ResultMessageEnumerator); - ReadPromise = new(); RowDescription = new(); } @@ -38,6 +36,11 @@ public void Reset() } } + internal readonly struct ReadPromiseState() + { + public ValueTaskSourcePromise Promise { get; } = new(); + } + // The value wrapper lets ReadState and CommandResult share one MessageEnumerator instance // without an interface or another adapter allocation. internal readonly struct ResultMessageEnumerator() : IEnumerator, IAsyncEnumerator diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index ea980b3..2eff2c3 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -345,7 +345,8 @@ FlowTasks ExecuteAutoCore(Context context) // the flow, preserving single-writer tenure without blocking reads behind socket backpressure. return new FlowTasks( trailingExecutionTask: writeTask, - pipelineTask: DispatchPipelinedRead(context, context.GetProtocolStatic().ReadPromise)); + pipelineTask: DispatchPipelinedRead( + context, context.GetProtocolStatic().Promise)); } // Defer state-machine creation until activation because all flows share one protocol-static promise. diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index e11b029..cf11c42 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1543,7 +1543,9 @@ sealed class PipelineSlots( } } - internal sealed class Control(PgClientProtocol protocol, bool poolFacing) : IProtocolStatic + internal sealed class Control(PgClientProtocol protocol, bool poolFacing) : + IProtocolStatic, + IProtocolStatic { // The pipeline whose slots this Control reads, bound right after that pipeline is created. The // outer (pool-facing) Control reads the protocol's own pipeline; an exclusive flow's inner @@ -1909,5 +1911,8 @@ internal void OnIdle() CommandFlow.ReadState _commandFlowReadState = new(); ref readonly CommandFlow.ReadState IProtocolStatic.Value => ref _commandFlowReadState; + readonly CommandFlow.ReadPromiseState _commandFlowReadPromiseState = new(); + ref readonly CommandFlow.ReadPromiseState IProtocolStatic.Value + => ref _commandFlowReadPromiseState; } } From 8c66217285c38a512996335a2396827dc8fc4d23 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 04:12:51 +0200 Subject: [PATCH 060/136] Derive deferred command read state from its context --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 2eff2c3..a11452f 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -127,10 +127,8 @@ bool IsFullyConsumedFinalResult // Pipelined dispatch state. Lives here (not on PgClientFlow base) because the shared-promise // optimization that needs these fields is CommandFlow-specific (see DispatchPipelinedRead). Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _executePipelinedCore; - ValueTaskSourcePromise? _pipelinePromise; Context _context; bool _contextPublished; - ValueTask _task; // Consumer terminality, body terminality, and framework release are distinct phases. Start and // pre-start termination race during shutdown, so one atomic state owns that decision. const int BodyNotStarted = 0; @@ -379,7 +377,6 @@ ValueTask DispatchPipelinedRead(Context context, ValueTaskSourcePromise pr } } - _pipelinePromise = promise; // Static continuation: a bridge into framework state, so no captured scheduling context is needed. waiter.OnCompleted(static state => { @@ -394,20 +391,21 @@ ValueTask DispatchPipelinedRead(Context context, ValueTaskSourcePromise pr catch (Exception ex) { flow._executePipelinedCore.SetException(ex); } return; } - var promise = flow._pipelinePromise!; + var promise = ctx.GetProtocolStatic().Promise; PromiseAsyncValueTaskMethodBuilder.Promise = promise; ValueTask task = flow.ExecutePipelined(ctx); try { if (!task.IsCompleted) { - flow._task = task; ((IValueTaskSource)promise).OnCompleted(static state => { var flow = (CommandFlow)state!; try { - flow._task.GetAwaiter().GetResult(); + var promise = flow._context + .GetProtocolStatic().Promise; + ((IValueTaskSource)promise).GetResult(promise.Token); flow._executePipelinedCore.SetResult(true); } catch (Exception ex) @@ -1211,10 +1209,8 @@ protected override void OnReset() _drainModeEntered = false; WaitForDrainOnDispose = true; // Dispatch state is per-tenure. - _pipelinePromise = null; _contextPublished = false; _context = default; - _task = default; _bodyState = BodyNotStarted; _consumerAdvanced = false; } From 2a9c5599800c4e7e01df665afc9874ad6d6632b4 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 04:45:44 +0200 Subject: [PATCH 061/136] Dispatch command result publication through the flow --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 39 ++++++++++++++++++++++++--- Slon/Pg/Protocol/PgClientFlow.cs | 13 ++++++++- Slon/Pg/Protocol/PgClientProtocol.cs | 2 ++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index a11452f..fd55d0d 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -135,6 +135,14 @@ bool IsFullyConsumedFinalResult const int BodyRunning = 1; const int BodyTerminated = 2; int _bodyState; + DetachedPublication _detachedPublication; + + enum DetachedPublication : byte + { + None, + Result, + Completion + } sealed class CancellationState { @@ -867,8 +875,7 @@ void SetResult(CommandResult? next) CompleteEnumeration(); } if (publishAsync) - context.SubmitDetached(static state => ((CommandFlow)state!) - .CompleteEnumeration(runContinuationsAsynchronously: false), this); + SubmitPublication(context, DetachedPublication.Completion); return; } if (publishAsync) @@ -876,8 +883,7 @@ void SetResult(CommandResult? next) // Queue the publication itself so the body reaches its next caller gate before user code // resumes. Routing through the protocol scheduler preserves that ordering without forcing // every result continuation onto the ThreadPool. - context.SubmitDetached(static state => ((CommandFlow)state!) - .TrySetEnumeratorResult(true, runContinuationsAsynchronously: false), this); + SubmitPublication(context, DetachedPublication.Result); } else TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); @@ -898,6 +904,31 @@ static void ReadRfq(PgDecoder decoder) } } + void SubmitPublication(Context context, DetachedPublication publication) + { + Debug.Assert(_detachedPublication is DetachedPublication.None); + _detachedPublication = publication; + context.SubmitDetached((IThreadPoolWorkItem)this); + } + + private protected override void ExecuteDetachedWorkItem() + { + var publication = _detachedPublication; + _detachedPublication = DetachedPublication.None; + switch (publication) + { + case DetachedPublication.Result: + TrySetEnumeratorResult(true, runContinuationsAsynchronously: false); + break; + case DetachedPublication.Completion: + CompleteEnumeration(runContinuationsAsynchronously: false); + break; + default: + ThrowHelper.ThrowInvalidOperation("The command flow has no publication pending."); + break; + } + } + void SetCallerCancellationToken(CancellationToken token) { var cancellation = GetOrCreateCancellationState(); diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 7158616..2c5f547 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -137,7 +137,11 @@ internal void PrepareActivationDispatch(PgClientProtocol.Control control) void IThreadPoolWorkItem.Execute() { var control = _pendingActivationControl; - Debug.Assert(control is not null); + if (control is null) + { + ExecuteDetachedWorkItem(); + return; + } _pendingActivationControl = null; // The decoder bind already ran synchronously at activation; this dispatch is only the body // wake. Skip it for a flow the abort retired before the dispatch ran: its activation source is @@ -147,6 +151,9 @@ void IThreadPoolWorkItem.Execute() control!.Activate(this); } + private protected virtual void ExecuteDetachedWorkItem() + => ThrowHelper.ThrowInvalidOperation("The flow has no detached work item pending."); + readonly bool _supportsDeferredFlush; internal bool SupportsDeferredFlush => _supportsDeferredFlush; int _rfqCount; @@ -551,6 +558,8 @@ internal ValueTask WaitForCancellationAttempt() internal void SubmitDetached(Action action, object? state, bool preferLocal = true) => _executionControl.SubmitDetached(action, state, preferLocal); + internal void SubmitDetached(IThreadPoolWorkItem workItem, bool preferLocal = true) + => _executionControl.SubmitDetached(workItem, preferLocal); internal void RequestBackendCancellation(PgClientFlow instigator, int window, BackendCancellationTiming timing, TaskCompletionSource? delivery, object episodeKey, int scope, @@ -736,6 +745,8 @@ internal readonly struct ExecutionControl(PgClientFlow flow, PgClientProtocol.Co internal void SubmitDetached(Action action, object? state, bool preferLocal = true) => control.SubmitDetached(action, state, preferLocal); + internal void SubmitDetached(IThreadPoolWorkItem workItem, bool preferLocal = true) + => control.SubmitDetached(workItem, preferLocal); // Small optimization to allow us to skip the final sync message if we can piggyback on the flow's final rfq. public bool LastMessageInducesRfq => flow._lastMessageInducesRfq; diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index cf11c42..b29603d 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1571,6 +1571,8 @@ public void BindPipeline( public ValueTask FlushAsync(CancellationToken cancellationToken) => protocol.FlushAsync(cancellationToken); internal void SubmitDetached(Action action, object? state, bool preferLocal = true) => protocol._activationScheduler.SubmitDetached(action, state, preferLocal); + internal void SubmitDetached(IThreadPoolWorkItem workItem, bool preferLocal = true) + => protocol._activationScheduler.SubmitDetached(workItem, preferLocal); PgClientFlow? _cancellationActivatedFlow; internal (PgClientFlow? Owner, int Window) CancellationActivation { From ed3ac85e380d39e7a6a40976a5a68eeec10cc23f Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 06:19:11 +0200 Subject: [PATCH 062/136] Keep command error handling off the read frame --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 147 ++++++++++++++------------ 1 file changed, 82 insertions(+), 65 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index fd55d0d..8396a20 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -582,26 +582,8 @@ await _commands.ItemRef(_commandIndex) MarkBodyInitiatedDrain(); } - CommandResult result; - { - ref readonly var readState = ref context.GetProtocolStatic(); - readState.ResultMessageEnumerator.Initialize(_commands.ItemRef(_commandIndex), _decoder); - result = _enumeratorCurrent ?? readState.CommandResult; - - ref readonly var resultCommand = ref _commands.ItemRef(_commandIndex); - var descriptor = resultCommand.Descriptor; - // We were preparing and we have no error from parse, make a prepared descriptor. - if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault - && (_pgError is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) - { - descriptor = CommandDescriptor.CreatePrepared( - descriptor.CommandName, - describeForPreparation ? describedParameterTypes : descriptor.ParameterTypes, - _requestedRowDescription?.Preserve()); - } - result.Initialize(this, _commandIndex, descriptor, _requestedRowDescription, - !resultCommand.DescribeOnly, resultCommand.IsSimple(), _pgError); - } + var result = InitializeResult( + context, describeForPreparation, describedParameterTypes); ((CommandFlowObserver?)GetObserver(out var observerState)) ?.OnCommandResult(this, result, observerState); @@ -711,43 +693,10 @@ await _commands.ItemRef(_commandIndex) completeError = resultEnumerator.CompleteError; } - var resultErrorIsOwnCancellation = result.Error is { } resultError - && IsOwnCancellation(resultError); - if (suppressEnumeration && result.Error is { } suppressedError - && !resultErrorIsOwnCancellation) - { - (_drainErrors ??= new()).Add(PgErrorException.Create(suppressedError)); - capturedThisCommand = true; - if (!IsDraining) - MarkBodyInitiatedDrain(); - } - - { - // Accumulate each command's fresh error while draining, but do not duplicate an error - // already captured during its read phase or delivered to a live consumer. - var completeErrorIsOwnCancellation = completeError is { } completedWithError - && IsOwnCancellation(completedWithError.Error); - if ((consumeInternally || IsConsumingNonQuery || IsDraining && !_isResultReady) - && !capturedThisCommand && completeError is { } err - && !completeErrorIsOwnCancellation) - (_drainErrors ??= new()).Add(PgErrorException.Create(err.Error)); - } - - // Extended-query errors discard every following command through the next Sync. Skip - // those commands locally and consume the RFQ which is their only wire response. - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) - { - while (++_commandIndex < CommandCount && !_commands[_commandIndex].WithSync) { } - - if (IsAsync) - await ReadRfqAsync(_decoder).ConfigureAwait(false); - else - ReadRfq(_decoder); - - // Reaching the end means the discarded segment terminated at our appended Sync. - if (_commandIndex == CommandCount) - _readFlowRfq = false; - } + if (result.Error is not null || completeError is not null) + await HandleCommandErrorsAsync( + result, suppressEnumeration, consumeInternally, + capturedThisCommand, completeError).ConfigureAwait(false); } // The framework observes trailing write failure before releasing this flow. @@ -889,19 +838,87 @@ void SetResult(CommandResult? next) TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); } - async ValueTask ReadRfqAsync(PgDecoder decoder) + } + + static async ValueTask ReadRfqAsync(PgDecoder decoder) + { + var message = await decoder.GetNextAsync().ConfigureAwait(false); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } + + static void ReadRfq(PgDecoder decoder) + { + var message = decoder.GetNext(); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + CommandResult InitializeResult( + Context context, bool describeForPreparation, + ParameterTypeList describedParameterTypes) + { + ref readonly var readState = ref context.GetProtocolStatic(); + readState.ResultMessageEnumerator.Initialize(_commands.ItemRef(_commandIndex), _decoder!); + var result = _enumeratorCurrent ?? readState.CommandResult; + + ref readonly var command = ref _commands.ItemRef(_commandIndex); + var descriptor = command.Descriptor; + // We were preparing and we have no error from parse, make a prepared descriptor. + if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault + && (_pgError is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) { - var message = await decoder.GetNextAsync().ConfigureAwait(false); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); + descriptor = CommandDescriptor.CreatePrepared( + descriptor.CommandName, + describeForPreparation ? describedParameterTypes : descriptor.ParameterTypes, + _requestedRowDescription?.Preserve()); } + result.Initialize(this, _commandIndex, descriptor, _requestedRowDescription, + !command.DescribeOnly, command.IsSimple(), _pgError); + return result; + } - static void ReadRfq(PgDecoder decoder) + async ValueTask HandleCommandErrorsAsync( + CommandResult result, bool suppressEnumeration, bool consumeInternally, + bool capturedThisCommand, + (PgError Error, TransactionStatus TransactionStatus)? completeError) + { + var resultErrorIsOwnCancellation = result.Error is { } resultError + && IsOwnCancellation(resultError); + if (suppressEnumeration && result.Error is { } suppressedError + && !resultErrorIsOwnCancellation) { - var message = decoder.GetNext(); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); + (_drainErrors ??= new()).Add(PgErrorException.Create(suppressedError)); + capturedThisCommand = true; + if (!IsDraining) + MarkBodyInitiatedDrain(); } + + // Accumulate each command's fresh error while draining, but do not duplicate an error + // already captured during its read phase or delivered to a live consumer. + var completeErrorIsOwnCancellation = completeError is { } completedWithError + && IsOwnCancellation(completedWithError.Error); + if ((consumeInternally || IsConsumingNonQuery || IsDraining && !_isResultReady) + && !capturedThisCommand && completeError is { } error + && !completeErrorIsOwnCancellation) + (_drainErrors ??= new()).Add(PgErrorException.Create(error.Error)); + + // Extended-query errors discard every following command through the next Sync. Skip + // those commands locally and consume the RFQ which is their only wire response. + if (completeError is not { TransactionStatus: TransactionStatus.Unknown }) + return; + + while (++_commandIndex < CommandCount && !_commands[_commandIndex].WithSync) { } + + if (IsAsync) + await ReadRfqAsync(_decoder!).ConfigureAwait(false); + else + ReadRfq(_decoder!); + + // Reaching the end means the discarded segment terminated at our appended Sync. + if (_commandIndex == CommandCount) + _readFlowRfq = false; } void SubmitPublication(Context context, DetachedPublication publication) From 635a483144b0f3b4608eaece9eb815d32b37e017 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 06:19:11 +0200 Subject: [PATCH 063/136] Keep generic pipe reads off the direct entry frame --- Slon/Pg/Protocol/PgDecoder.cs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 37daeb1..4eae929 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -545,16 +545,7 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau } } - var readTask = pipe.ReadAsync(readToken); - if (!readTask.IsCompletedSuccessfully) - return MoveNextAsyncCore(readTask, null, null, cancellationToken, frontierFlow); - LeaveCancellationReadFrontier(frontierFlow); - if (CompleteRead( - readTask.Result, _cancellationTokenSource.Token, - out var readCompleted)) - continue; - if (readCompleted) - return new(ReadCompleted()); + return BeginPipeRead(pipe, readToken, cancellationToken, frontierFlow); } catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) { @@ -574,6 +565,16 @@ public ValueTask MoveNextAsync(CancellationToken cancellationToken = defau } + [MethodImpl(MethodImplOptions.NoInlining)] + ValueTask BeginPipeRead( + ProtocolReadPipe pipe, + CancellationToken readToken, + CancellationToken cancellationToken, + PgClientFlow frontierFlow) + => MoveNextAsyncCore( + pipe.ReadAsync(readToken), null, null, + cancellationToken, frontierFlow); + [MethodImpl(MethodImplOptions.NoInlining)] ValueTask AwaitDirectRead( ValueTask directReadTask, From 9a3a9392242c4b615edc20c39ca8323cb42128e0 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 06:22:06 +0200 Subject: [PATCH 064/136] Move result publication out of the command state machine --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 93 +++++++++++++-------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 8396a20..be43eaa 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -623,7 +623,7 @@ await _commands.ItemRef(_commandIndex) // Result continuations run asynchronously so the body can reach the next gate // before user code asks for the next result. Buffered batches then advance inline // from MoveNextAsync instead of suspending one Task state machine per result. - SetResult(result); + PublishEnumeratorResult(context, result); if (!IsDraining && !IsConsumingNonQuery) { @@ -718,7 +718,7 @@ await HandleCommandErrorsAsync( } } - SetResult(null); + PublishEnumeratorResult(context, null); } catch (PgClientClosedException) when (context.IsProtocolClosed) { @@ -729,7 +729,7 @@ await HandleCommandErrorsAsync( if (IsDraining) { if (!IsEnumerationCompleted) - SetResult(null); + PublishEnumeratorResult(context, null); return; } CompleteEnumerationWithException(context.FlowTerminationException); @@ -787,57 +787,56 @@ await HandleCommandErrorsAsync( readState.Reset(); PublishBodyTerminated(); } - void SetResult(CommandResult? next) - { - var completed = next is null; - var publishAsync = IsAsync; - if (completed) - { - _enumeratorCurrent = null; - } - else - { - if (Volatile.Read(ref _cancellationState) is { } cancellation) - cancellation.CallerToken = default; + } - if (!ReferenceEquals(_enumeratorCurrent, next)) - _enumeratorCurrent = next; + void PublishEnumeratorResult(Context context, CommandResult? next) + { + var completed = next is null; + var publishAsync = IsAsync; + if (completed) + { + _enumeratorCurrent = null; + } + else + { + if (Volatile.Read(ref _cancellationState) is { } cancellation) + cancellation.CallerToken = default; - } + if (!ReferenceEquals(_enumeratorCurrent, next)) + _enumeratorCurrent = next; + } - // Close is durable across generations; complete the current one without publishing a result. - if (_callerInteractionCore.CloseException is not null) - { - CompleteEnumerationWithClose(_callerInteractionCore.CloseException); - return; - } - if (completed) + // Close is durable across generations; complete the current one without publishing a result. + if (_callerInteractionCore.CloseException is not null) + { + CompleteEnumerationWithClose(_callerInteractionCore.CloseException); + return; + } + if (completed) + { + // Publish durable terminal state atomically with respect to consumer rearming. Async + // consumers complete from the protocol scheduler so they cannot reenter this lock or + // the pipeline frame that still owns the shared promise; sync consumers retain their + // caller-driven completion. + using (_rearmLock.EnterScope()) { - // Publish durable terminal state atomically with respect to consumer rearming. Async - // consumers complete from the protocol scheduler so they cannot reenter this lock or - // the pipeline frame that still owns the shared promise; sync consumers retain their - // caller-driven completion. - using (_rearmLock.EnterScope()) - { - PublishEnumerationCompleted(); - if (!publishAsync) - CompleteEnumeration(); - } - if (publishAsync) - SubmitPublication(context, DetachedPublication.Completion); - return; + PublishEnumerationCompleted(); + if (!publishAsync) + CompleteEnumeration(); } if (publishAsync) - { - // Queue the publication itself so the body reaches its next caller gate before user code - // resumes. Routing through the protocol scheduler preserves that ordering without forcing - // every result continuation onto the ThreadPool. - SubmitPublication(context, DetachedPublication.Result); - } - else - TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); + SubmitPublication(context, DetachedPublication.Completion); + return; } - + if (publishAsync) + { + // Queue the publication itself so the body reaches its next caller gate before user code + // resumes. Routing through the protocol scheduler preserves that ordering without forcing + // every result continuation onto the ThreadPool. + SubmitPublication(context, DetachedPublication.Result); + } + else + TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); } static async ValueTask ReadRfqAsync(PgDecoder decoder) From 4af258780c42ded885f8eba42dff044c883b6c95 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 06:24:36 +0200 Subject: [PATCH 065/136] Keep timeout draining off the command read frame --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 63 +++++++++++++++------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index be43eaa..089007f 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using System.Threading.Tasks.Sources; using Slon.Runtime.CompilerServices; // A unique result type distinguishes the caller gate from this flow's other IValueTaskSource instantiations. @@ -742,35 +743,7 @@ await HandleCommandErrorsAsync( } catch (TimeoutException ex) { - CompleteEnumerationWithException(ex); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, - BackendCancellationTiming.AtReadFrontier, allowCompletedEnumeration: true); - if (context.IsProtocolClosed) - throw; - - // The timeout is terminal for the consumer, not for the body. Keep ownership of the - // command sequence and drain every remaining RFQ window; reaching each RFQ requests - // cancellation for the next window through OnCancellationWindowCompleted. Recovery is - // reserved for a failure of this semantic drain, where only wire obligations remain. - if (Volatile.Read(ref _cancellationState) is { } cancellation) - await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); - ((CommandFlowObserver?)GetObserver(out var observerState)) - ?.OnDrainStarted(this, observerState); - try - { - while (context.OutstandingRfqCount != 0) - _ = await _decoder!.GetNextAuto().ConfigureAwait(false); - } - catch (TimeoutException) - { - // The semantic drain owns the same cancellation episode. A timeout here would - // otherwise bypass the outer catch and leave the episode unaware that its first - // read-timeout escalation produced no protocol progress. - RequestCancel(default, CancellationScope.RemainingFlow, - BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier, - allowCompletedEnumeration: true); - throw; - } + await HandleTimeoutAsync(context, ex).ConfigureAwait(false); return; } catch (Exception ex) @@ -839,6 +812,38 @@ void PublishEnumeratorResult(Context context, CommandResult? next) TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); } + async ValueTask HandleTimeoutAsync(Context context, TimeoutException exception) + { + CompleteEnumerationWithException(exception); + RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, + BackendCancellationTiming.AtReadFrontier, allowCompletedEnumeration: true); + if (context.IsProtocolClosed) + ExceptionDispatchInfo.Throw(exception); + + // The timeout is terminal for the consumer, not for the body. Keep ownership of the + // command sequence and drain every remaining RFQ window; reaching each RFQ requests + // cancellation for each following window through the cancellation coordinator. Recovery is + // reserved for a failure of this semantic drain, where only wire obligations remain. + if (Volatile.Read(ref _cancellationState) is { } cancellation) + await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); + ((CommandFlowObserver?)GetObserver(out var observerState)) + ?.OnDrainStarted(this, observerState); + try + { + while (context.OutstandingRfqCount != 0) + _ = await _decoder!.GetNextAuto().ConfigureAwait(false); + } + catch (TimeoutException) + { + // The semantic drain owns the same cancellation episode. A timeout here would otherwise + // leave the episode unaware that its first read-timeout escalation made no protocol progress. + RequestCancel(default, CancellationScope.RemainingFlow, + BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier, + allowCompletedEnumeration: true); + throw; + } + } + static async ValueTask ReadRfqAsync(PgDecoder decoder) { var message = await decoder.GetNextAsync().ConfigureAwait(false); From 6e929241fb58fa77fb38a2b0a2abc5f3c33c892e Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 06:33:51 +0200 Subject: [PATCH 066/136] Keep resumable writes off the async execution frame --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 31 ++++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 089007f..e56303f 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -319,27 +319,18 @@ FlowTasks ExecuteAutoCore(Context context) // Writes are independent of consumer admission. Inter-result gates provide backpressure. // Async writes use transport completion; sync writes use the resumable non-blocking path so // the caller thread retains execution ownership across readiness waits. - var encoder = IsAsync ? default : context.GetEncoder(); var appendSync = !_commands[CommandCount - 1].WithSync; _readFlowRfq = appendSync; - if (IsAsync) - { - // Caller cancellation never cancels wire I/O. A partially cancelled write requires - // protocol recovery and can strand already-pipelined successors; the body instead - // observes the latched intent and drains every written command to RFQ. - writeTask = _commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default); - } - else - { - using (encoder.BeginResumableWriteScope()) - writeTask = _commands.WriteCommandsResumable(encoder, appendSync); - } + // Caller cancellation never cancels wire I/O. A partially cancelled write requires + // protocol recovery and can strand already-pipelined successors; the body instead + // observes the latched intent and drains every written command to RFQ. + writeTask = IsAsync + ? _commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) + : WriteCommandsResumable(context, appendSync); // Observe synchronous faults here; pending writes remain the framework-owned trailing task. if (writeTask.IsCompleted) writeTask.GetAwaiter().GetResult(); - else if (!IsAsync) - writeTask = encoder.RunResumableTask(writeTask); } catch (Exception ex) { @@ -356,6 +347,16 @@ FlowTasks ExecuteAutoCore(Context context) context, context.GetProtocolStatic().Promise)); } + [MethodImpl(MethodImplOptions.NoInlining)] + ValueTask WriteCommandsResumable(Context context, bool appendSync) + { + var encoder = context.GetEncoder(); + ValueTask writeTask; + using (encoder.BeginResumableWriteScope()) + writeTask = _commands.WriteCommandsResumable(encoder, appendSync); + return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); + } + // Defer state-machine creation until activation because all flows share one protocol-static promise. ValueTask DispatchPipelinedRead(Context context, ValueTaskSourcePromise promise) { From 95cb4891e49000b299151a6a89e3ee09efee32c3 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 06:38:36 +0200 Subject: [PATCH 067/136] Fuse single prepared command-list writes --- Slon/Pg/Protocol/Flows/CommandExtensions.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Slon/Pg/Protocol/Flows/CommandExtensions.cs b/Slon/Pg/Protocol/Flows/CommandExtensions.cs index f989dcc..151831c 100644 --- a/Slon/Pg/Protocol/Flows/CommandExtensions.cs +++ b/Slon/Pg/Protocol/Flows/CommandExtensions.cs @@ -16,7 +16,6 @@ public static bool IsSimple(this in Command command) => static bool CanWritePreparedExecution(in Command command, in CommandDescriptor descriptor) => descriptor.IsPrepared && command.Parameters.Count is 0 && descriptor.ParameterTypes.Count is 0 && command.ResultFormats.Length is 0; - [MethodImpl(MethodImplOptions.AggressiveInlining)] static ValueTask WritePreparedExecutionAsync( in Command command, in CommandDescriptor descriptor, PgEncoder encoder, bool appendSync, @@ -36,6 +35,15 @@ static ValueTask WritePreparedExecutionAsync( public static ValueTask WriteCommandsAsync(this CommandList commands, PgEncoder encoder, bool appendSync, CancellationToken cancellationToken = default) { + if (commands.Count is 1) + { + var command = commands[0]; + var descriptor = command.Descriptor; + if (CanWritePreparedExecution(command, descriptor)) + return WritePreparedExecutionAsync( + command, descriptor, encoder, appendSync, cancellationToken); + } + for (var i = 0; i < commands.Count; i++) { var command = commands[i]; From cd5009a00ac71e161532949d968662a326e7ed88 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 06:43:35 +0200 Subject: [PATCH 068/136] Avoid command copies in prepared write scans --- Slon/Pg/Protocol/Flows/CommandExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandExtensions.cs b/Slon/Pg/Protocol/Flows/CommandExtensions.cs index 151831c..7f0f2b4 100644 --- a/Slon/Pg/Protocol/Flows/CommandExtensions.cs +++ b/Slon/Pg/Protocol/Flows/CommandExtensions.cs @@ -37,7 +37,7 @@ public static ValueTask WriteCommandsAsync(this CommandList commands, PgEncoder { if (commands.Count is 1) { - var command = commands[0]; + ref readonly var command = ref commands.ItemRef(0); var descriptor = command.Descriptor; if (CanWritePreparedExecution(command, descriptor)) return WritePreparedExecutionAsync( @@ -46,7 +46,7 @@ public static ValueTask WriteCommandsAsync(this CommandList commands, PgEncoder for (var i = 0; i < commands.Count; i++) { - var command = commands[i]; + ref readonly var command = ref commands.ItemRef(i); var descriptor = command.Descriptor; if (!descriptor.IsPrepared || command.Parameters.Count is not 0 || descriptor.ParameterTypes.Count is not 0 || command.ResultFormats.Length is not 0) From 6ee49f8d8dd3f6d309bf5837d738168cf9c550b2 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 07:19:00 +0200 Subject: [PATCH 069/136] Keep preparation reads off the command state machine --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 55 ++++++++++++++++++--------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index e56303f..5e4ea0b 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -29,6 +29,11 @@ public readonly struct CommandFlowOptions [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] public partial class CommandFlow : PgClientFlow, IValueTaskSource, IValueTaskSource, IValueTaskSource { + sealed class PreparationReadState + { + internal ParameterTypeList ParameterTypes; + } + static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); internal override bool DefersSyncHandoff => true; @@ -495,22 +500,11 @@ async ValueTask ExecutePipelined(Context context) if (!IsDraining && context.IsProtocolClosed) throw context.FlowTerminationException; - ParameterTypeList describedParameterTypes = default; + PreparationReadState? preparationRead = null; if (describeForPreparation) { - var rowDescription = context.GetProtocolStatic().RowDescription; - if (IsAsync) - { - (_pgError, describedParameterTypes, _requestedRowDescription) = - await _commands.ItemRef(_commandIndex) - .ReadPreparationDescriptionAsync(_decoder, rowDescription).ConfigureAwait(false); - } - else - { - (_pgError, describedParameterTypes, _requestedRowDescription) = - _commands.ItemRef(_commandIndex) - .ReadPreparationDescription(_decoder, rowDescription); - } + preparationRead = new(); + await ReadPreparationDescription(context, preparationRead).ConfigureAwait(false); } else if (IsAsync && hasPreparedDescription) { @@ -585,7 +579,7 @@ await _commands.ItemRef(_commandIndex) } var result = InitializeResult( - context, describeForPreparation, describedParameterTypes); + context, preparationRead); ((CommandFlowObserver?)GetObserver(out var observerState)) ?.OnCommandResult(this, result, observerState); @@ -861,8 +855,7 @@ static void ReadRfq(PgDecoder decoder) [MethodImpl(MethodImplOptions.NoInlining)] CommandResult InitializeResult( - Context context, bool describeForPreparation, - ParameterTypeList describedParameterTypes) + Context context, PreparationReadState? preparationRead) { ref readonly var readState = ref context.GetProtocolStatic(); readState.ResultMessageEnumerator.Initialize(_commands.ItemRef(_commandIndex), _decoder!); @@ -876,7 +869,7 @@ CommandResult InitializeResult( { descriptor = CommandDescriptor.CreatePrepared( descriptor.CommandName, - describeForPreparation ? describedParameterTypes : descriptor.ParameterTypes, + preparationRead?.ParameterTypes ?? descriptor.ParameterTypes, _requestedRowDescription?.Preserve()); } result.Initialize(this, _commandIndex, descriptor, _requestedRowDescription, @@ -884,6 +877,32 @@ CommandResult InitializeResult( return result; } + ValueTask ReadPreparationDescription(Context context, PreparationReadState state) + { + var rowDescription = context.GetProtocolStatic().RowDescription; + ref readonly var command = ref _commands.ItemRef(_commandIndex); + if (!IsAsync) + { + (_pgError, state.ParameterTypes, _requestedRowDescription) = + command.ReadPreparationDescription(_decoder!, rowDescription); + return default; + } + + var read = command.ReadPreparationDescriptionAsync(_decoder!, rowDescription); + if (!read.IsCompletedSuccessfully) + return AwaitRead(this, state, read); + (_pgError, state.ParameterTypes, _requestedRowDescription) = read.Result; + return default; + + static async ValueTask AwaitRead( + CommandFlow flow, PreparationReadState state, + ValueTask<(PgError?, ParameterTypeList, RowDescription?)> read) + { + (flow._pgError, state.ParameterTypes, flow._requestedRowDescription) = + await read.ConfigureAwait(false); + } + } + async ValueTask HandleCommandErrorsAsync( CommandResult result, bool suppressEnumeration, bool consumeInternally, bool capturedThisCommand, From 9c9e48f056d6a1fd89ffec863c29b5fc1162e020 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 07:28:26 +0200 Subject: [PATCH 070/136] Keep internal result draining off the reader frame --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 54 +++++++++++++++++---------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 5e4ea0b..c315952 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -595,8 +595,7 @@ async ValueTask ExecutePipelined(Context context) CompleteEnumerationWithClose(close); MarkBodyInitiatedDrain(); } - var consumeInternally = IsConsumingNonQuery || suppressEnumeration; - if (!IsDraining && !consumeInternally) + if (!IsDraining && !IsConsumingNonQuery && !suppressEnumeration) { // Eager async execution must wait for the consumer to arm generation zero before // publishing its first result. Synchronous execution already runs on that caller. @@ -657,24 +656,11 @@ async ValueTask ExecutePipelined(Context context) // remaining rows. Re-read the current execution mode after every resumption. (PgError Error, TransactionStatus TransactionStatus)? completeError; // Consumption mode may change while the body is suspended; use the current value. - if (consumeInternally || IsConsumingNonQuery) + if (IsConsumingNonQuery || suppressEnumeration) { - while (_decoder.Current.Header.Type is PgTypes.BackendType.DataRow) - { - if (!_decoder.TryMoveNext()) - await _decoder.GetNextAsync().ConfigureAwait(false); - } - result.CompleteNonQuery(_decoder.Current); - var completion = _commands.ItemRef(_commandIndex).CompleteAsync(_decoder); - completeError = await completion.ConfigureAwait(false); - if (_pgError is null && completeError is null) - { - var recordsAffected = result.GetCommandComplete().BatchRecordsAffected; - if (recordsAffected >= 0) - _nonQueryRecordsAffected = _nonQueryRecordsAffected < 0 - ? recordsAffected - : checked(_nonQueryRecordsAffected + recordsAffected); - } + await CompleteInternalConsumptionAsync( + result, suppressEnumeration, capturedThisCommand).ConfigureAwait(false); + continue; } else if (IsAsync) { @@ -691,7 +677,7 @@ async ValueTask ExecutePipelined(Context context) if (result.Error is not null || completeError is not null) await HandleCommandErrorsAsync( - result, suppressEnumeration, consumeInternally, + result, suppressEnumeration, consumeInternally: false, capturedThisCommand, completeError).ConfigureAwait(false); } @@ -903,6 +889,34 @@ static async ValueTask AwaitRead( } } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] + async ValueTask CompleteInternalConsumptionAsync( + CommandResult result, bool suppressEnumeration, bool capturedThisCommand) + { + while (_decoder!.Current.Header.Type is PgTypes.BackendType.DataRow) + { + if (!_decoder.TryMoveNext()) + await _decoder.GetNextAsync().ConfigureAwait(false); + } + result.CompleteNonQuery(_decoder.Current); + var completeError = await _commands.ItemRef(_commandIndex) + .CompleteAsync(_decoder).ConfigureAwait(false); + if (_pgError is null && completeError is null) + { + var recordsAffected = result.GetCommandComplete().BatchRecordsAffected; + if (recordsAffected >= 0) + _nonQueryRecordsAffected = _nonQueryRecordsAffected < 0 + ? recordsAffected + : checked(_nonQueryRecordsAffected + recordsAffected); + } + + if (result.Error is not null || completeError is not null) + await HandleCommandErrorsAsync( + result, suppressEnumeration, consumeInternally: true, + capturedThisCommand, completeError).ConfigureAwait(false); + } + async ValueTask HandleCommandErrorsAsync( CommandResult result, bool suppressEnumeration, bool consumeInternally, bool capturedThisCommand, From 8dcdaa6bf280cab72ae6c7e876b0c45c074ec510 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 07:39:38 +0200 Subject: [PATCH 071/136] Reuse buffered length while parsing messages --- Slon/Pg/Protocol/BackendMessageCursor.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index d975ec9..c221bb3 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -56,11 +56,12 @@ public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) { + var bufferedLength = _buffer.Length; if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) - && (_buffer.Length < Header.ByteCount + && (bufferedLength < Header.ByteCount || !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader))) { - _requiredBufferedLength = ConsumedLength + Header.ByteCount; + _requiredBufferedLength = _initialLength - bufferedLength + Header.ByteCount; buffer = default; bufferLength = default; header = default; @@ -74,16 +75,16 @@ internal bool TryReadNextBuffer(out BackendHeader header, var required = backendType is BackendType.DataRow ? Math.Min(messageLength, (uint)_dataRowStreamingThreshold) : messageLength; - if (_buffer.Length < required) + if (bufferedLength < required) { - _requiredBufferedLength = ConsumedLength + required; + _requiredBufferedLength = _initialLength - bufferedLength + required; buffer = default; bufferLength = default; header = default; return false; } - buffer = _buffer.SplitInPlace(Math.Min(_buffer.Length, messageLength)); + buffer = _buffer.SplitInPlace(Math.Min(bufferedLength, messageLength)); _requiredBufferedLength = 0; Debug.Assert(buffer.Length <= uint.MaxValue); bufferLength = unchecked((uint)buffer.Length); From 813167b409a3a3b3a831c83489cfd98cc76257d3 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 07:50:14 +0200 Subject: [PATCH 072/136] Keep pending stream flushes off the common write frame --- Slon/Pipelines/StreamPipeWriter.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Slon/Pipelines/StreamPipeWriter.cs b/Slon/Pipelines/StreamPipeWriter.cs index fc42a3e..ecfbaac 100644 --- a/Slon/Pipelines/StreamPipeWriter.cs +++ b/Slon/Pipelines/StreamPipeWriter.cs @@ -396,7 +396,11 @@ async ValueTask FlushAsyncCore(AutoResetCancellationTokenSource? to if (didWrite) { - await Stream.FlushAsync(token).ConfigureAwait(false); + var flush = Stream.FlushAsync(token); + if (!flush.IsCompletedSuccessfully) + await AwaitFlush(flush).ConfigureAwait(false); + else + flush.GetAwaiter().GetResult(); } } @@ -428,6 +432,11 @@ async ValueTask FlushAsyncCore(AutoResetCancellationTokenSource? to EndStartedFlush(); } } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] + static async ValueTask AwaitFlush(Task flush) + => await flush.ConfigureAwait(false); } } From bb6e78518042ceb260b44488c724253bc5ffd6a1 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 10:07:42 +0200 Subject: [PATCH 073/136] Avoid wait-handle allocation after flow completion --- Slon/Pg/Protocol/PgClientFlow.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 2c5f547..af50594 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -393,6 +393,12 @@ internal void WaitForCompleteSynchronously(CancellationToken cancellationToken = cancellationToken.ThrowIfCancellationRequested(); Volatile.Write(ref _completionWaiterPending, 1); var token = _completionCore.Version; + if (_completionCore.GetStatus(token) is not ValueTaskSourceStatus.Pending) + { + _ = ((IValueTaskSource)this).GetResult(token); + return; + } + var state = GetOrCreateOptionalState(); var completionEvent = state.CompletionEvent; if (completionEvent is null) From d9ec971327dc983c7634ff3457b23c0c321c8649 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 10:24:59 +0200 Subject: [PATCH 074/136] Keep direct-read completion out of the decoder frame --- Slon/Pg/Protocol/PgDecoder.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index 4eae929..ca53c19 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -188,6 +188,7 @@ bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask return false; } + [MethodImpl(MethodImplOptions.NoInlining)] bool CompleteDirectRead(int length, CancellationToken cancellationToken, out ValueTask next, out bool readFinished, out bool completed) { From 1cfce3438d7f7deabb0191a7c557cbf6faac4e38 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 12:50:25 +0200 Subject: [PATCH 075/136] Keep cursor buffer publication stack-only --- Slon/Pg/Protocol/BackendMessageContext.cs | 10 ++++++++-- Slon/Pg/Protocol/BackendMessageCursor.cs | 17 +++++++++-------- Slon/Runtime/CompilerServices/StackValue.cs | 8 ++++++++ 3 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 Slon/Runtime/CompilerServices/StackValue.cs diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 1e50db1..ac6fb43 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Slon.Pipelines; +using Slon.Runtime.CompilerServices; namespace Slon.Pg.Protocol; @@ -456,8 +457,11 @@ public bool TryMoveNext() PublishPeeked(); return true; } - if (!_cursor.TryReadNextBuffer(out var header, out var buffer, out var bufferLength)) + var bufferSlot = default(StackValue>); + if (!_cursor.TryReadNextBuffer( + out var header, ref bufferSlot, out var bufferLength)) return false; + var buffer = bufferSlot.Value; ResetMessageState(); if (bufferLength < header.MessageLength) _decoder.SetCurrentMessageLength( @@ -539,11 +543,13 @@ public bool TryPeekNext(out BackendHeader header) header = _current.Header; return true; } + var bufferSlot = default(StackValue>); if (!_cursor.TryReadNextBuffer( - out header, out var buffer, out var bufferLength)) + out header, ref bufferSlot, out var bufferLength)) { return false; } + var buffer = bufferSlot.Value; var messageLength = header.MessageLength; var buffered = bufferLength >= messageLength; if (!buffered) diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index c221bb3..b8c3c63 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Slon.Pipelines; +using Slon.Runtime.CompilerServices; using static Slon.Pg.Protocol.PgTypes; namespace Slon.Pg.Protocol; @@ -43,18 +44,19 @@ public readonly BackendMessageCursor Slice(long offset) [MethodImpl(MethodImplOptions.NoInlining)] public bool TryReadNextInPlace(out BackendHeader header, out ReadOnlySequence buffer, out uint bufferLength) { - if (!TryReadNextBuffer(out header, out var fastBuffer, out bufferLength)) + var bufferSlot = default(StackValue>); + if (!TryReadNextBuffer(out header, ref bufferSlot, out bufferLength)) { buffer = default; return false; } - buffer = fastBuffer.Sequence; + buffer = bufferSlot.Value.Sequence; return true; } [MethodImpl(MethodImplOptions.NoInlining)] internal bool TryReadNextBuffer(out BackendHeader header, - out FastReadOnlySequence buffer, out uint bufferLength) + ref StackValue> buffer, out uint bufferLength) { var bufferedLength = _buffer.Length; if (!Header.TryParse(_buffer.FirstSpan, out var protoHeader) @@ -62,7 +64,6 @@ internal bool TryReadNextBuffer(out BackendHeader header, || !Header.TryParseMultiSegment(_buffer.Sequence, out protoHeader))) { _requiredBufferedLength = _initialLength - bufferedLength + Header.ByteCount; - buffer = default; bufferLength = default; header = default; return false; @@ -78,17 +79,17 @@ internal bool TryReadNextBuffer(out BackendHeader header, if (bufferedLength < required) { _requiredBufferedLength = _initialLength - bufferedLength + required; - buffer = default; bufferLength = default; header = default; return false; } - buffer = _buffer.SplitInPlace(Math.Min(bufferedLength, messageLength)); + var result = _buffer.SplitInPlace(Math.Min(bufferedLength, messageLength)); _requiredBufferedLength = 0; - Debug.Assert(buffer.Length <= uint.MaxValue); - bufferLength = unchecked((uint)buffer.Length); + Debug.Assert(result.Length <= uint.MaxValue); + bufferLength = unchecked((uint)result.Length); header = BackendHeader.FromHeader(protoHeader); + buffer.Value = result; return true; } diff --git a/Slon/Runtime/CompilerServices/StackValue.cs b/Slon/Runtime/CompilerServices/StackValue.cs new file mode 100644 index 0000000..3431dd0 --- /dev/null +++ b/Slon/Runtime/CompilerServices/StackValue.cs @@ -0,0 +1,8 @@ +namespace Slon.Runtime.CompilerServices; + +// A byref to this wrapper is proven stack-only, allowing non-inlined callees to store +// reference-bearing values without checked write barriers. +ref struct StackValue(T value) where T : allows ref struct +{ + public T Value = value; +} From f0ed42180b893c875b5aaaab5ff4af34b5387af1 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 13:11:25 +0200 Subject: [PATCH 076/136] Cache segment-backed cursor representation --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 24 +++++ Slon/Pg/Protocol/BackendMessageCursor.cs | 101 +++++++++++------- 2 files changed, 86 insertions(+), 39 deletions(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index bc0940d..f29149a 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -31,6 +31,15 @@ public ReadOnlySequence To(SequenceSegment end) => new(this, 0, end, end.Memory.Length); } + sealed class TestMemoryManager(byte[] buffer) : MemoryManager + { + public override Span GetSpan() => buffer; + public override MemoryHandle Pin(int elementIndex = 0) + => throw new NotSupportedException(); + public override void Unpin() { } + protected override void Dispose(bool disposing) { } + } + sealed class RejectRetiredSuppliedReadReader(PipeReader inner) : PipeReader { ReadResult _activeRead; @@ -565,6 +574,21 @@ public void BackendCursor_FramesUnknownMessageType() Assert.AreEqual((BackendType)(byte)'o', header.Type); } + [TestMethod] + public void BackendCursor_FramesMemoryManagerBackedMessage() + { + using var manager = new TestMemoryManager( + BackendMessageBytes(BackendType.CommandComplete, 8)); + var cursor = new BackendMessageCursor( + new ReadOnlySequence(manager.Memory)); + + Assert.IsTrue(cursor.TryReadNextInPlace( + out var header, out var message, out var bufferedLength)); + Assert.AreEqual(BackendType.CommandComplete, header.Type); + Assert.AreEqual(8, message.Length); + Assert.AreEqual(8u, bufferedLength); + } + [TestMethod] public void BackendCursor_RejectsMessageBeyondPostgreSqlAllocationLimit() { diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index b8c3c63..d366b8f 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -111,6 +111,9 @@ public readonly bool TryReadNext(out BackendHeader header, out ReadOnlySequence< // reconstruct and rediscover the backing of a ReadOnlySequence. Materialize one only at API seams. internal struct FastReadOnlySequence { + const int SegmentFlag = int.MinValue; + const int IndexMask = int.MaxValue; + object? _startObject; object? _endObject; int _startIndex; @@ -118,13 +121,13 @@ internal struct FastReadOnlySequence long _length; FastReadOnlySequence(object? startObject, int startIndex, - object? endObject, int endIndex, long length) + object? endObject, int endIndex, long length, bool segmentBacked) { Debug.Assert(Unsafe.SizeOf>() is 32); _startObject = startObject; _endObject = endObject; - _startIndex = startIndex; - _endIndex = endIndex; + _startIndex = EncodeIndex(startIndex, segmentBacked); + _endIndex = EncodeIndex(endIndex, segmentBacked); _length = length; } @@ -133,37 +136,47 @@ public FastReadOnlySequence(ReadOnlySequence sequence) Debug.Assert(Unsafe.SizeOf>() is 32); _startObject = sequence.Start.GetObject(); _endObject = sequence.End.GetObject(); - _startIndex = sequence.Start.GetInteger() & int.MaxValue; - _endIndex = sequence.End.GetInteger() & int.MaxValue; + var segmentBacked = _startObject is ReadOnlySequenceSegment; + _startIndex = EncodeIndex( + sequence.Start.GetInteger() & IndexMask, segmentBacked); + _endIndex = EncodeIndex( + sequence.End.GetInteger() & IndexMask, segmentBacked); _length = sequence.Length; } + static int EncodeIndex(int index, bool segmentBacked) + => segmentBacked ? index | SegmentFlag : index; + + readonly bool IsSegmentBacked => _startIndex < 0; + public ReadOnlySequence Sequence { get { if (_startObject is null) return default; - if (_startObject is T[] array) + var startIndex = StartIndex; + var endIndex = EndIndex; + if (IsSegmentBacked) { - Debug.Assert(ReferenceEquals(_startObject, _endObject)); - return new(array, _startIndex, _endIndex - _startIndex); + return new((ReadOnlySequenceSegment)_startObject, startIndex, + (ReadOnlySequenceSegment)_endObject!, endIndex); } - if (_startObject is MemoryManager manager) + if (_startObject is T[] array) { Debug.Assert(ReferenceEquals(_startObject, _endObject)); - return new(manager.Memory.Slice( - _startIndex, _endIndex - _startIndex)); + return new(array, startIndex, endIndex - startIndex); } - return new((ReadOnlySequenceSegment)_startObject!, _startIndex, - (ReadOnlySequenceSegment)_endObject!, _endIndex); + var manager = (MemoryManager)_startObject; + Debug.Assert(ReferenceEquals(_startObject, _endObject)); + return new(manager.Memory.Slice(startIndex, endIndex - startIndex)); } } public long Length => _length; public object? StartObject => _startObject; public object? EndObject => _endObject; - public int StartIndex => _startIndex; - public int EndIndex => _endIndex; + public int StartIndex => _startIndex & IndexMask; + public int EndIndex => _endIndex & IndexMask; public ReadOnlySpan FirstSpan { @@ -171,37 +184,45 @@ public ReadOnlySpan FirstSpan { if (_startObject is null) return default; + var startIndex = StartIndex; + var endIndex = EndIndex; + if (IsSegmentBacked) + { + var memory = ((ReadOnlySequenceSegment)_startObject).Memory; + var end = ReferenceEquals(_startObject, _endObject) + ? endIndex + : memory.Length; + return memory.Span.Slice(startIndex, end - startIndex); + } if (_startObject is T[] array) { Debug.Assert(ReferenceEquals(_startObject, _endObject)); - return array.AsSpan(_startIndex, _endIndex - _startIndex); + return array.AsSpan(startIndex, endIndex - startIndex); } - var memory = _startObject is MemoryManager manager - ? manager.Memory - : ((ReadOnlySequenceSegment)_startObject).Memory; - var end = ReferenceEquals(_startObject, _endObject) - ? _endIndex - : memory.Length; - return memory.Span.Slice(_startIndex, end - _startIndex); + return ((MemoryManager)_startObject).Memory.Span + .Slice(startIndex, endIndex - startIndex); } } - ReadOnlyMemory FirstMemory => _startObject switch - { - T[] array => array, - MemoryManager manager => manager.Memory, - ReadOnlySequenceSegment segment => segment.Memory, - _ => throw new UnreachableException() - }; + ReadOnlyMemory FirstMemory + => IsSegmentBacked + ? ((ReadOnlySequenceSegment)_startObject!).Memory + : _startObject switch + { + T[] array => array, + MemoryManager manager => manager.Memory, + _ => throw new UnreachableException() + }; // Returns the sequence before the index, stores the sequence after it in place. [MethodImpl(MethodImplOptions.AggressiveInlining)] public FastReadOnlySequence SplitInPlace(long offset) { + var startIndex = StartIndex; var firstEnd = ReferenceEquals(_startObject, _endObject) - ? _endIndex + ? EndIndex : FirstMemory.Length; - var firstLength = firstEnd - _startIndex; + var firstLength = firstEnd - startIndex; if (offset == _length) { var exhausted = this; @@ -211,22 +232,24 @@ public FastReadOnlySequence SplitInPlace(long offset) return exhausted; } if (offset == firstLength - && _startObject is ReadOnlySequenceSegment segment - && segment.Next is { } next) + && IsSegmentBacked + && ((ReadOnlySequenceSegment)_startObject!).Next is { } next) { var boundaryPrefix = new FastReadOnlySequence( - segment, _startIndex, segment, firstEnd, offset); + _startObject, startIndex, _startObject, firstEnd, offset, + segmentBacked: true); _startObject = next; - _startIndex = 0; + _startIndex = SegmentFlag; _length -= offset; return boundaryPrefix; } if ((ulong)offset < (uint)firstLength) { - var splitIndex = _startIndex + (int)offset; + var splitIndex = startIndex + (int)offset; var prev = new FastReadOnlySequence( - _startObject, _startIndex, _startObject, splitIndex, offset); - _startIndex = splitIndex; + _startObject, startIndex, _startObject, splitIndex, offset, + IsSegmentBacked); + _startIndex = EncodeIndex(splitIndex, IsSegmentBacked); _length -= offset; return prev; } From 38a8c51f2352f66f4c72a0c5780c636f4a0e17ac Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 03:04:18 +0200 Subject: [PATCH 077/136] Keep net11 protocol reads reusable --- Slon/Pg/Protocol/PgDecoder.cs | 50 +----------------------------- Slon/Pipelines/StreamPipeReader.cs | 12 ------- 2 files changed, 1 insertion(+), 61 deletions(-) diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index ca53c19..ff06ad6 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -19,9 +19,7 @@ namespace Slon.Pg.Protocol; public sealed class PgDecoder: IEnumerator, IAsyncEnumerator { readonly ProtocolReadPipe _pipe; -#if !NET11_0_OR_GREATER readonly StreamPipeReader? _directReader; -#endif readonly CancellationToken _abortToken; readonly TimeSpan _defaultReadTimeout; readonly Action? _readTimeoutArmed; @@ -89,9 +87,7 @@ TimeSpan GetRemainingTimeout() Action? readTimeoutArmed) { _pipe = pipe; -#if !NET11_0_OR_GREATER _directReader = pipe.PipeReader as StreamPipeReader; -#endif _abortToken = abortToken; _defaultReadTimeout = defaultReadTimeout; _readTimeout = defaultReadTimeout; @@ -176,7 +172,6 @@ bool ReadNext(TimeSpan timeout) return _pipe.MoveNext(timeout); } -#if !NET11_0_OR_GREATER bool TryBeginDirectRead(CancellationToken cancellationToken, out ValueTask task) { if (_directReader is { SupportsDirectRead: true } directReader) @@ -203,7 +198,6 @@ bool CompleteDirectRead(int length, CancellationToken cancellationToken, } void AbortDirectRead() => _directReader!.AbortDirectRead(); -#endif // Builds a scope-bound shell over the shared pipe with the scope's abort token. internal static PgDecoder CreateScopeShell(PgDecoder baseShell, CancellationToken abortToken, TimeSpan defaultReadTimeout) => new(baseShell._pipe, abortToken, defaultReadTimeout, baseShell._readTimeoutArmed); @@ -492,7 +486,6 @@ internal bool IsAtCancellationReadFrontier(PgClientFlow flow, int window) /// Flow-owned cancellation path for a parked read. Without it the only break-out is protocol /// abort. An uncaught firing triggers the protocol's recovery path, so prefer a /// coordination-boundary check in connection-preserving flows. -#if !NET11_0_OR_GREATER public ValueTask MoveNextAsync(CancellationToken cancellationToken = default) { EnsureUsableCts(); @@ -586,6 +579,7 @@ ValueTask AwaitDirectRead( : MoveNextDirectAsync(directReadTask, frontierFlow); [MethodImpl(MethodImplOptions.NoInlining)] + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask MoveNextDirectAsync( ValueTask directReadTask, @@ -694,28 +688,18 @@ async ValueTask MoveNextDirectWithCancellationAsync( } } -#endif -#if NET11_0_OR_GREATER - public async ValueTask MoveNextAsync(CancellationToken cancellationToken = default) - { - EnsureUsableCts(); - var pipe = _pipe; - PgClientFlow? frontierFlow = null; -#else [MethodImpl(MethodImplOptions.NoInlining)] async ValueTask MoveNextAsyncCore(ValueTask? readTask, ValueTask? directReadTask, ValueTask? messageHandledTask, CancellationToken cancellationToken, PgClientFlow? frontierFlow = null) { var pipe = _pipe; -#endif var timeoutSet = false; var registration = cancellationToken.UnsafeRegister(static (state, _) => ((CancellationTokenSource)state!).Cancel(), _cancellationTokenSource); try { while (true) { -#if !NET11_0_OR_GREATER if (messageHandledTask is { } t) { if (!await t.ConfigureAwait(false)) @@ -752,9 +736,7 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, throw TranslateEof(ex); } } -#endif -#if !NET11_0_OR_GREATER if (directReadTask is { } pendingDirectRead) { try @@ -801,15 +783,10 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, throw; } } -#endif while (TryMoveNext(pipe)) { var handleTask = CurrentExecutionControl.HandleMessageAuto(pipe.Current); -#if NET11_0_OR_GREATER - if (!await handleTask.ConfigureAwait(false)) - return true; -#else if (!handleTask.IsCompletedSuccessfully) { messageHandledTask = handleTask; @@ -817,12 +794,9 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, } if (!handleTask.Result) return true; -#endif } -#if !NET11_0_OR_GREATER if (messageHandledTask.HasValue) continue; -#endif PrepareRead(); @@ -830,33 +804,13 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, { var token = _cancellationTokenSource.Token; frontierFlow = EnterCancellationReadFrontier(); -#if NET11_0_OR_GREATER - var nextRead = pipe.ReadAsync(token); - if (!nextRead.IsCompletedSuccessfully && !timeoutSet) - { - ArmReadTimeout(); - timeoutSet = true; - } - var result = await nextRead.ConfigureAwait(false); - LeaveCancellationReadFrontier(frontierFlow); - frontierFlow = null; - if (CompleteRead(result, token, out var readCompleted)) - continue; - if (readCompleted) - return ReadCompleted(); -#else if (TryBeginDirectRead(token, out var nextDirectRead)) directReadTask = nextDirectRead; else readTask = pipe.ReadAsync(token); -#endif } catch (Exception ex) when (_cancellationTokenSource.IsCancellationRequested) { throw TranslateReadCancellation(ex, cancellationToken); } -#if NET11_0_OR_GREATER - catch (EndOfStreamException ex) - { throw TranslateEof(ex); } -#endif } } finally @@ -868,9 +822,7 @@ async ValueTask MoveNextAsyncCore(ValueTask? readTask, SetRemainingTimeout(Timeout.InfiniteTimeSpan); } } -#if !NET11_0_OR_GREATER } -#endif bool ReadCompleted() { diff --git a/Slon/Pipelines/StreamPipeReader.cs b/Slon/Pipelines/StreamPipeReader.cs index eee1ee5..418e580 100644 --- a/Slon/Pipelines/StreamPipeReader.cs +++ b/Slon/Pipelines/StreamPipeReader.cs @@ -12,10 +12,8 @@ interface IStreamOwner : IDisposable, IAsyncDisposable { } abstract class StreamPipeReader : PipeReader { -#if !NET11_0_OR_GREATER readonly ValueTaskSourcePromise _readAsyncCorePromise = new(); bool _directReadAwaitingData; -#endif readonly IStreamOwner? _streamOwner; int _isReadActive; int _readerCompleted; @@ -231,7 +229,6 @@ internal void EnsureCanUpgradeStream() throw new InvalidOperationException("The reader must be open, idle, and empty before its stream can be upgraded."); } -#if !NET11_0_OR_GREATER // Direct reads retain reader tenure until their continuation returns through CompleteDirectRead // or AbortDirectRead. The protocol must interrupt and join that tenure before completing the // reader and returning its destination buffer. @@ -308,7 +305,6 @@ ValueTask StartDataRead(CancellationToken cancellationToken) var buffer = Segments.Reserve(0, enforceHint: false); return Stream.ReadAsync(buffer, cancellationToken); } -#endif protected ReadResult ReadCore(int minimumSize, TimeSpan timeout) { @@ -381,11 +377,6 @@ protected ReadResult ReadCore(int minimumSize, TimeSpan timeout) } -#if NET11_0_OR_GREATER - protected async ValueTask ReadAsyncCore(int minimumSize, CancellationToken cancellationToken) - { - var tokenSource = PendingReadTokenSource; -#else protected ValueTask ReadAsyncCore(int minimumSize, CancellationToken cancellationToken) { PromiseAsyncValueTaskMethodBuilder.Promise = _readAsyncCorePromise; @@ -403,7 +394,6 @@ protected ValueTask ReadAsyncCore(int minimumSize, CancellationToken async ValueTask ReadAsyncCore(int minimumSize, AutoResetCancellationTokenSource? tokenSource, CancellationToken cancellationToken) { -#endif // Cancellation token was already checked before getting here. if (!TryStartRead()) ThrowAlreadyReading(); @@ -462,9 +452,7 @@ async ValueTask ReadAsyncCore(int minimumSize, } } } -#if !NET11_0_OR_GREATER } -#endif [MethodImpl(MethodImplOptions.AggressiveInlining)] int GetReadSizeHint(int minimumSize) From 3b80f0dc68d01b805a97c8e2b676d38d5a7dec74 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 01:49:46 +0200 Subject: [PATCH 078/136] Remove unused flow RFQ callback --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 3 --- Slon/Pg/Protocol/PgClientFlow.cs | 3 --- 2 files changed, 6 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index c315952..47137f9 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -1144,9 +1144,6 @@ void RequestBackendCancellation(BackendCancellationTiming timing = BackendCancel } } - protected override void OnCancellationWindowCompleted(int completedWindow, int remainingWindowCount) - { } - bool IsOwnCancellation(PgError error) { if (Volatile.Read(ref _cancellationState) is not { } cancellation diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index af50594..259ee9c 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -485,7 +485,6 @@ protected virtual void OnStopping(Exception exception) {} /// No overridable hook may run after that signal: a completion waiter can immediately Reset and /// enqueue the same object for its next tenure. protected virtual void OnReleasing(Exception? exception) {} - protected virtual void OnCancellationWindowCompleted(int completedWindow, int remainingWindowCount) {} protected virtual void OnDiscarded() {} protected virtual void OnReset() {} @@ -838,7 +837,6 @@ internal bool TryHandleKnownMessage(BackendMessage backendMessage, out bool hand flow._rfqCount -= 1; var completedWindow = flow._cancellationWindow++; control.OnFlowRfq(flow, backendMessage, completedWindow, flow._rfqCount); - flow.OnCancellationWindowCompleted(completedWindow, flow._rfqCount); handled = false; return true; case PgTypes.BackendType.NoticeResponse: @@ -866,7 +864,6 @@ ValueTask HandleMessageAutoCore(BackendMessage backendMessage) flow._rfqCount -= 1; var completedWindow = flow._cancellationWindow++; control.OnFlowRfq(flow, backendMessage, completedWindow, flow._rfqCount); - flow.OnCancellationWindowCompleted(completedWindow, flow._rfqCount); goto default; case PgTypes.BackendType.NoticeResponse: // We sink all notices (this includes RAISE notices) and expect those to end up on the flow for user retrieval/logging. From e4f917262c52a43fbc4b81a95abfbab8a3546edf Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 03:09:24 +0200 Subject: [PATCH 079/136] Preserve pooling builders under runtime async --- Slon/Pg/CommandResult.cs | 3 +++ Slon/Pg/Protocol/BackendMessageBodyReader.cs | 3 +++ Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs | 1 + Slon/Pg/Protocol/Flows/CommandFlow.cs | 3 +++ Slon/Pg/Protocol/PgClientFlowSource.cs | 1 + Slon/Pg/Protocol/PgDecoder.cs | 1 + Slon/Pg/Row.cs | 1 + Slon/Pg/Serialization/PgStreamingConverter.cs | 1 + Slon/Pipelines/PipeOutputWriter.cs | 1 + Slon/Pipelines/StreamPipeWriter.cs | 1 + Slon/SlonDataReader.cs | 2 ++ 11 files changed, 18 insertions(+) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index 53b84f7..8bf6791 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -247,6 +247,7 @@ internal ValueTask CompleteAsync() } [MethodImpl(MethodImplOptions.NoInlining)] + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask CompleteAsyncCore() { @@ -559,6 +560,7 @@ bool HandleUncommon(in BackendMessage.Accessor current) } [MethodImpl(MethodImplOptions.NoInlining)] + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask MoveNextAsyncCore(ValueTask task) { @@ -588,6 +590,7 @@ async ValueTask MoveNextAsyncCore(ValueTask task) } [MethodImpl(MethodImplOptions.NoInlining)] + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask BufferRowAsync(ValueTask task, Row row) { diff --git a/Slon/Pg/Protocol/BackendMessageBodyReader.cs b/Slon/Pg/Protocol/BackendMessageBodyReader.cs index 1acaf70..7cc40f2 100644 --- a/Slon/Pg/Protocol/BackendMessageBodyReader.cs +++ b/Slon/Pg/Protocol/BackendMessageBodyReader.cs @@ -74,6 +74,7 @@ public ValueTask ReadAsync(CancellationToken cancellationToken = default) } return Core(task); + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(ValueTask task) => Publish(await task.ConfigureAwait(false)); @@ -105,6 +106,7 @@ public ValueTask ExtendAsync(CancellationToken cancellationToken = default) } return Core(task); + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(ValueTask task) => Publish(_context.CompleteExtend( @@ -176,6 +178,7 @@ public ValueTask BufferAllAsync(CancellationToken cancellationToken = default) return default; return Core(cancellationToken); + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(CancellationToken cancellationToken) { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 3a177bc..4c8c3f3 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -152,6 +152,7 @@ public ValueTask MoveNextAsync() return Core(); + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask Core() { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 47137f9..82850dd 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -223,6 +223,7 @@ public CommandFlow Initialize(bool async, in CommandFlowOptions options) // makes the ownership rule structural: no enumerator is exposed on this path, so mixing // enumeration with internal consumption is unrepresentable. The declaration precedes any // consumer-side gate release, so the body observes it at first wake and never publishes. + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] internal async ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) { @@ -289,6 +290,7 @@ protected override ValueTask ExecuteAuto(Context context) return new(ExecuteAutoCore(context)); } + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask ExecuteAfterHandoff(Context context) { @@ -1016,6 +1018,7 @@ void RegisterCancellationCallbacksLocked(CancellationState cancellation) } } + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask DisposeCancellationRegistrations(CancellationState cancellation) { diff --git a/Slon/Pg/Protocol/PgClientFlowSource.cs b/Slon/Pg/Protocol/PgClientFlowSource.cs index f65d521..0bbeb6e 100644 --- a/Slon/Pg/Protocol/PgClientFlowSource.cs +++ b/Slon/Pg/Protocol/PgClientFlowSource.cs @@ -543,6 +543,7 @@ WaitForNextAwaitable WaitCore() // Only reached on real write backpressure (the flush didn't complete inline), so a pooled // box is plenty - the promise-reuse builder would be overkill for how rarely this fires. + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask FlushThenWaitAsync(ValueTask flushTask) { diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index ff06ad6..a3855e6 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -1010,6 +1010,7 @@ public ValueTask GetNextAsync() return default; } + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask GetNextAsyncCore(ValueTask task) { diff --git a/Slon/Pg/Row.cs b/Slon/Pg/Row.cs index d34903e..57e45d9 100644 --- a/Slon/Pg/Row.cs +++ b/Slon/Pg/Row.cs @@ -532,6 +532,7 @@ internal ValueTask BufferAllAsync(CancellationToken cancellationToken = default) return default; return Core(cancellationToken); + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask Core(CancellationToken token) { diff --git a/Slon/Pg/Serialization/PgStreamingConverter.cs b/Slon/Pg/Serialization/PgStreamingConverter.cs index f6c4f3b..8272c75 100644 --- a/Slon/Pg/Serialization/PgStreamingConverter.cs +++ b/Slon/Pg/Serialization/PgStreamingConverter.cs @@ -67,6 +67,7 @@ static class PgStreamingConverterHelpers { // Split out from the generic class to amortize the huge size penalty per async state machine, which would otherwise be per // instantiation. + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] public static async ValueTask AwaitTask(Task task, Continuation continuation) { diff --git a/Slon/Pipelines/PipeOutputWriter.cs b/Slon/Pipelines/PipeOutputWriter.cs index 95e1b2b..2820065 100644 --- a/Slon/Pipelines/PipeOutputWriter.cs +++ b/Slon/Pipelines/PipeOutputWriter.cs @@ -42,6 +42,7 @@ ValueTask IOutputWriter.FlushAsync(CancellationToken cancellationToken) EnsureFlushed(flushTask.Result); return new(); + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] static async ValueTask Core(ValueTask flushTask) => EnsureFlushed(await flushTask.ConfigureAwait(false)); diff --git a/Slon/Pipelines/StreamPipeWriter.cs b/Slon/Pipelines/StreamPipeWriter.cs index ecfbaac..417561c 100644 --- a/Slon/Pipelines/StreamPipeWriter.cs +++ b/Slon/Pipelines/StreamPipeWriter.cs @@ -175,6 +175,7 @@ ValueTask IOutputWriter.FlushAsync(CancellationToken cancellationToken) EnsureFlushed(flushTask.Result); return default; + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] static async ValueTask Core(ValueTask flushTask) => EnsureFlushed(await flushTask.ConfigureAwait(false)); diff --git a/Slon/SlonDataReader.cs b/Slon/SlonDataReader.cs index 52a42dd..67cffd7 100644 --- a/Slon/SlonDataReader.cs +++ b/Slon/SlonDataReader.cs @@ -451,6 +451,7 @@ void DisposeEnumerator(out bool ownsCleanup) } } + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask DisposeEnumeratorAsync() { @@ -572,6 +573,7 @@ void CloseCore(bool resetForReuse) } } + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask CloseAsyncCore(bool resetForReuse) { From 25cc1958e8c7beede1aa64de8dd7a27697699726 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 00:57:58 +0200 Subject: [PATCH 080/136] Correct complete-write and reader completion invariants --- Slon/Pg/Protocol/PgEncoder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Slon/Pg/Protocol/PgEncoder.cs b/Slon/Pg/Protocol/PgEncoder.cs index fd6f9f9..4ff7e55 100644 --- a/Slon/Pg/Protocol/PgEncoder.cs +++ b/Slon/Pg/Protocol/PgEncoder.cs @@ -268,7 +268,7 @@ internal static void WritePreparedExecutionCore(ProtocolDataWriter writer, Encod + (describe ? header + describeBody : 0) + (execute ? header + executeBody : 0) + syncCount * header); - var span = writer.GetCompleteMessagesSpan(total); + var span = writer.GetCompleteMessagesSpan(total).Slice(0, total); WriteHeader(span, FrontendType.Bind, bindBody); span[header] = 0; // unnamed portal From d89d9baabbfe3101c3605a472793e3b3973d2d46 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:01:48 +0200 Subject: [PATCH 081/136] Cover command flow lifecycle and row collection --- Slon.Tests/Pg/CommandFlowContractTests.cs | 369 +++++++++++++++++++++ Slon.Tests/Pg/CommandResultCollectTests.cs | 89 +++++ 2 files changed, 458 insertions(+) create mode 100644 Slon.Tests/Pg/CommandFlowContractTests.cs create mode 100644 Slon.Tests/Pg/CommandResultCollectTests.cs diff --git a/Slon.Tests/Pg/CommandFlowContractTests.cs b/Slon.Tests/Pg/CommandFlowContractTests.cs new file mode 100644 index 0000000..54488a6 --- /dev/null +++ b/Slon.Tests/Pg/CommandFlowContractTests.cs @@ -0,0 +1,369 @@ +using Slon.Pg; +using Slon.Pg.Protocol; +using Slon.Pg.Protocol.Flows; +using Slon.Pg.Serialization; +using Slon.Pg.Types; +using Slon.Text; + +namespace Slon.Tests.Pg; + +// Command-result semantics shared by consumer-driven PostgreSQL flows. +[TestClass] +public class CommandFlowContractTests +{ + static async Task Prepare( + PgClientProtocol protocol, string sql, EncodedCString name) + { + var results = protocol.Queue(new CommandFlow(async: true, + Command.Create(sql, commandName: name) with { DescribeOnly = true })).GetAsyncEnumerator(); + CommandDescriptor descriptor = default; + while (await results.MoveNextAsync()) + descriptor = results.Current.GetMetadata().ToPreparedDescriptor(); + await results.DisposeAsync(); + return descriptor; + } + + static Results Queue(PgClientProtocol protocol, in Command command, + CancellationToken cancellationToken = default) + => new(protocol.Queue(new CommandFlow(async: true, command), cancellationToken) + .GetAsyncEnumerator()); + + static async Task CountRows(Results results) + { + var count = 0; + while (await results.MoveNextAsync()) + { + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync()) + count++; + await rows.DisposeAsync(); + } + await results.DisposeAsync(); + return count; + } + + [ConnectionCreatingTestMethod] + [DataRow(0)] + [DataRow(3)] + public async Task Prepared_NaturalExhaustion(int rowCount) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var descriptor = await Prepare(protocol, + $"select generate_series(1, {rowCount})", $"contract_rows_{rowCount}"); + + Assert.AreEqual(rowCount, + await CountRows(Queue(protocol, Command.Create(descriptor)))); + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task Unprepared_NaturalExhaustion() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + + Assert.AreEqual(2, await CountRows(Queue(protocol, + Command.Create("select generate_series(1, 2)")))); + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task DisposeBeforeAnyRead_DrainsAndKeepsWire() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var descriptor = await Prepare(protocol, + "select generate_series(1, 1000)", "contract_unread"); + var results = Queue(protocol, Command.Create(descriptor)); + + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task DisposeAfterOneRow_DrainsAndKeepsWire() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var descriptor = await Prepare(protocol, + "select generate_series(1, 20000)", "contract_partial"); + var results = Queue(protocol, Command.Create(descriptor)); + + Assert.IsTrue(await results.MoveNextAsync()); + var rows = results.Current.GetAsyncEnumerator(); + Assert.IsTrue(await rows.MoveNextAsync()); + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task CommandError_IsResultAndKeepsWire() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var descriptor = await Prepare(protocol, "select 1 / 0", "contract_error"); + var results = Queue(protocol, Command.Create(descriptor)); + + Assert.IsTrue(await results.MoveNextAsync()); + var failed = results.Current; + var rows = failed.GetAsyncEnumerator(); + Assert.IsFalse(await rows.MoveNextAsync()); + await rows.DisposeAsync(); + Assert.ThrowsExactly(() => failed.GetCommandComplete()); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task PreparedMetadataAndCompletion_Agree() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var descriptor = await Prepare(protocol, "select 42::int4", "contract_metadata"); + var results = Queue(protocol, Command.Create(descriptor)); + + Assert.IsTrue(await results.MoveNextAsync()); + var result = results.Current; + var metadata = result.GetMetadata(); + Assert.IsTrue(metadata.IsPrepared); + Assert.AreEqual(descriptor.CommandName, metadata.CommandName); + Assert.IsNotNull(metadata.RowDescription); + var rows = result.GetAsyncEnumerator(); + Assert.IsTrue(await rows.MoveNextAsync()); + Assert.AreEqual(42, rows.Current.GetReader().Read()); + Assert.IsFalse(await rows.MoveNextAsync()); + await rows.DisposeAsync(); + Assert.IsTrue(result.IsComplete); + Assert.AreEqual(StatementType.Select, result.GetCommandComplete().StatementType); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task CancellationWhileReadPending_DeliversTokenAndKeepsWire() + { + await using var protocol = await NewCancelableProtocolAsync(); + var descriptor = await Prepare(protocol, "select pg_sleep(30)", "contract_cancel_pending"); + using var cancellation = new CancellationTokenSource(); + var results = Queue(protocol, Command.Create(descriptor)); + + var pending = results.MoveNextAsync(cancellation.Token); + Assert.IsFalse(pending.IsCompleted); + cancellation.CancelAfter(TimeSpan.FromMilliseconds(200)); + var exception = await Assert.ThrowsExactlyAsync( + async () => await pending); + Assert.AreEqual(cancellation.Token, exception.CancellationToken); + await Assert.ThrowsExactlyAsync( + async () => await results.MoveNextAsync()); + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task CancellationAfterRow_DrainsAndKeepsWire() + { + await using var protocol = await NewCancelableProtocolAsync(); + var descriptor = await Prepare(protocol, + "select generate_series(1, 20000)", "contract_cancel_row"); + using var cancellation = new CancellationTokenSource(); + var results = Queue(protocol, Command.Create(descriptor)); + + Assert.IsTrue(await results.MoveNextAsync(cancellation.Token)); + var rows = results.Current.GetAsyncEnumerator(); + Assert.IsTrue(await rows.MoveNextAsync()); + cancellation.Cancel(); + await Assert.ThrowsExactlyAsync( + async () => await results.MoveNextAsync(cancellation.Token)); + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task PreCancelledRead_ReleasesCallerAndKeepsWire() + { + await using var protocol = await NewCancelableProtocolAsync(); + var descriptor = await Prepare(protocol, + "select generate_series(1, 1000)", "contract_precancel"); + var results = Queue(protocol, Command.Create(descriptor)); + var cancellationToken = new CancellationToken(canceled: true); + + var exception = await Assert.ThrowsExactlyAsync( + async () => await results.MoveNextAsync(cancellationToken)); + Assert.AreEqual(cancellationToken, exception.CancellationToken); + await Assert.ThrowsExactlyAsync( + async () => await results.MoveNextAsync(cancellationToken)); + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task SuccessorProgressesAfterAbandonment() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var descriptor = await Prepare(protocol, + "select generate_series(1, 20000)", "contract_successor"); + var first = Queue(protocol, Command.Create(descriptor)); + var second = Queue(protocol, Command.Create(descriptor)); + + Assert.IsTrue(await first.MoveNextAsync()); + await first.DisposeAsync(); + Assert.AreEqual(20000, await CountRows(second)); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task GracefulStopDrainsHeldResultAndFaultsConsumer() + { + var protocol = await PgTestPool.NewIsolatedAsync(options => + options.HeartbeatInterval = TimeSpan.FromMilliseconds(20)); + var descriptor = await Prepare(protocol, + "select generate_series(1, 1000)", "contract_graceful"); + var results = Queue(protocol, Command.Create(descriptor)); + Assert.IsTrue(await results.MoveNextAsync()); + + await protocol.CompleteAsync().WaitAsync(TimeSpan.FromSeconds(5)); + await Assert.ThrowsAsync( + async () => await results.MoveNextAsync()); + await results.DisposeAsync(); + } + + [ConnectionCreatingTestMethod(connections: 2)] + public async Task BackendTermination_IsCollateral() + { + await using var protocols = await PgTestPool.NewIsolatedProtocolsAsync(2); + var killer = protocols.Items[1]; + + var exception = await Terminate(); + Assert.IsInstanceOfType(exception); + + async Task Terminate() + { + await using var victim = await PgTestPool.NewIsolatedAsync(); + var pid = await ReadBackendPid(victim); + var descriptor = await Prepare(victim, "select pg_sleep(10)", + "contract_terminate_command"); + var results = Queue(victim, Command.Create(descriptor)); + var pending = results.MoveNextAsync(); + Assert.IsFalse(pending.IsCompleted); + await PgTestPool.RunAsync(killer, $"select pg_terminate_backend({pid})"); + var exception = await Assert.ThrowsAsync(async () => await pending); + await Assert.ThrowsAsync(async () => await results.DisposeAsync()); + await victim.Completion; + return exception; + } + } + + [ConnectionCreatingTestMethod] + public async Task TornTrailingWrite_RecoversWire() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var results = Queue(protocol, TornStreamedBind()); + + Exception? observed = null; + try + { + while (await results.MoveNextAsync()) + { + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync()) { } + await rows.DisposeAsync(); + results.Current.GetCommandComplete(); + } + } + catch (Exception exception) + { + observed = exception; + } + try + { + await results.DisposeAsync(); + } + catch (Exception exception) + { + observed ??= exception; + } + Assert.IsNotNull(observed); + + await PgTestPool.RunAsync(protocol, "select 42::int4"); + } + + static Task NewCancelableProtocolAsync() + => PgTestPool.NewIsolatedAsync(options => + options.CancelSender = PgTestPool.CreateCancelSender(PgTestPool.NewOptions())); + + static async Task ReadBackendPid(PgClientProtocol protocol) + { + var results = Queue(protocol, Command.Create("select pg_backend_pid()")); + var pid = 0; + while (await results.MoveNextAsync()) + { + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync()) + pid = rows.Current.GetReader().Read(); + await rows.DisposeAsync(); + } + await results.DisposeAsync(); + return pid; + } + + internal static Command TornStreamedBind() + { + var serializerOptions = new PgSerializerOptions(PgTypeCatalog.Default); + var value = new SlonParameter(new ThrowingReadStream(256 * 1024, 64 * 1024)); + var parameters = new SlonParameters { value }; + parameters.GetOrResolveTypeInfo( + 0, serializerOptions, preparedTypeId: null, allowUnspecified: false); + var parameterSource = new ParameterSource(parameters, SerializerParameterWriter.Instance); + return Command.Create( + "select octet_length($1::bytea)", new ParameterTypeList(parameterSource)) with + { + Parameters = parameterSource + }; + } + + sealed class ThrowingReadStream(int length, int throwAfter) : Stream + { + int _position; + + public override int Read(Span buffer) + { + if (_position >= throwAfter) + throw new IOException("Synthetic parameter read failure."); + var count = Math.Min(buffer.Length, throwAfter - _position); + buffer[..count].Clear(); + _position += count; + return count; + } + + public override ValueTask ReadAsync( + Memory buffer, CancellationToken cancellationToken = default) + => new(Read(buffer.Span)); + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => length; + public override long Position { get => _position; set => throw new NotSupportedException(); } + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) + => throw new NotSupportedException(); + public override int Read(byte[] buffer, int offset, int count) + => Read(buffer.AsSpan(offset, count)); + } + + readonly struct Results(CommandFlow.Enumerator inner) : IAsyncDisposable + { + internal CommandResult Current => inner.Current; + internal ValueTask MoveNextAsync(CancellationToken cancellationToken = default) + => inner.MoveNextAsync(cancellationToken); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + +} + diff --git a/Slon.Tests/Pg/CommandResultCollectTests.cs b/Slon.Tests/Pg/CommandResultCollectTests.cs new file mode 100644 index 0000000..6931208 --- /dev/null +++ b/Slon.Tests/Pg/CommandResultCollectTests.cs @@ -0,0 +1,89 @@ +using Slon.Pg; +using Slon.Pg.Protocol; +using Slon.Pg.Protocol.Flows; + +namespace Slon.Tests.Pg; + +[TestClass] +public sealed class CommandResultCollectTests +{ + [ConnectionCreatingTestMethod] + public async Task CollectsValueRowsAndLeavesWireReusable() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var values = new List(); + var results = protocol.Queue(new CommandFlow(async: true, + Command.Create("select generate_series(1, 100)"))).GetAsyncEnumerator(); + + Assert.IsTrue(await results.MoveNextAsync()); + await results.Current.CollectRowsAsync(values, + static (items, row) => items.Add(row.GetInt32(0))); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + + CollectionAssert.AreEqual(Enumerable.Range(1, 100).ToArray(), values); + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task BuffersAStreamingRowBeforeCallingCollector() + { + await using var protocol = await PgTestPool.NewIsolatedAsync( + options => options.DataRowStreamingThreshold = 1); + var values = new List<(int Id, string Text)>(); + var results = protocol.Queue(new CommandFlow(async: true, + Command.Create("select 42, repeat('x', 100000)"))).GetAsyncEnumerator(); + + Assert.IsTrue(await results.MoveNextAsync()); + await results.Current.CollectRowsAsync(values, + static (items, row) => items.Add((row.GetInt32(0), row.GetValue(1)))); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + + Assert.AreEqual(1, values.Count); + Assert.AreEqual(42, values[0].Id); + Assert.AreEqual(100000, values[0].Text.Length); + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task CollectorFailureDrainsBeforeRethrowing() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var callbacks = 0; + var results = protocol.Queue(new CommandFlow(async: true, + Command.Create("select generate_series(1, 100)"))).GetAsyncEnumerator(); + + Assert.IsTrue(await results.MoveNextAsync()); + var exception = await Assert.ThrowsExactlyAsync(async () => + await results.Current.CollectRowsAsync(0, (_, _) => + { + callbacks++; + throw new InvalidOperationException("collector failure"); + })); + Assert.AreEqual("collector failure", exception.Message); + Assert.AreEqual(1, callbacks); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + + await PgTestPool.RunAsync(protocol, "select 1"); + } + + [ConnectionCreatingTestMethod] + public async Task CannotCollectAfterRowEnumerationStarted() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var results = protocol.Queue(new CommandFlow(async: true, + Command.Create("select generate_series(1, 2)"))).GetAsyncEnumerator(); + + Assert.IsTrue(await results.MoveNextAsync()); + var rows = results.Current.GetAsyncEnumerator(); + Assert.IsTrue(await rows.MoveNextAsync()); + await Assert.ThrowsExactlyAsync(async () => + await results.Current.CollectRowsAsync(0, static (_, _) => { })); + await rows.DisposeAsync(); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + } +} + From 00f54f8ace6a82590e1ba78559e84cb8537080ce Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 09:25:45 +0200 Subject: [PATCH 082/136] Add the command execution flow foundation --- .../Pg/Protocol/Flows/CommandExecutionFlow.cs | 1053 +++++++++++++++++ Slon/Pg/Protocol/PgClientFlow.cs | 6 + 2 files changed, 1059 insertions(+) create mode 100644 Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs diff --git a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs new file mode 100644 index 0000000..d3a441e --- /dev/null +++ b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs @@ -0,0 +1,1053 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks.Sources; +using Slon.Runtime.CompilerServices; + +namespace Slon.Pg.Protocol.Flows; + +// Replacement-flow prototype: one consumer-owned decoder lifecycle for synchronous and asynchronous +// execution, with the general multi-command result shape. Kept internal until it covers the complete +// CommandFlow contract; the eventual single-command specialization will remain a separate sealed type. +internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource, IValueTaskSource +{ + // Decoder ownership. Reading and Draining name a frame that owns the decoder. Initial and + // ResultReady are idle states which exactly one party leaves by compare-exchange. + const int PhaseInitial = 0; + const int PhaseReading = 1; + const int PhaseResultReady = 2; + const int PhaseDraining = 3; + const int PhaseCompleted = 4; + int _phase; + + readonly CommandList _commands; + readonly TimeSpan? _pendingTimeout; + readonly Action? _resultObserver; + readonly object? _resultObserverState; + int _commandIndex = -1; + Context _context; + CommandResult? _current; + bool _readFlowRfq; + // Set by the consumer once it has started reading, so a drain knows whether to publish nothing. + bool _consumerDetached; + bool _consumerObservedCompletion; + + // Completed once the request is written and activation settled, faulted by teardown before then. + Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _readySource; + int _readyCompletion; + // The framework's pipeline task, completed by whichever frame consumes RFQ. + Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _pipelineTaskSource; + + // The submission token occupies this slot until a consumer attaches, after which the consumer + // token replaces it, including with default. Cancellation delivery, close, and failures stay cold. + CancellationToken _flowToken; + CancellationTokenRegistration _flowRegistration; + ColdState? _coldState; + FlowHandoffEvent? _handoffEvent; + bool _syncHandoffClaimed; + + sealed class ColdState + { + internal bool CancelRequested; + internal CancellationToken DeliverToken; + internal Exception? CloseException; + // Replayed by later consumer calls once the flow reached its terminal. + internal Exception? TerminalException; + // A command error observed while draining without a consumer. + internal Exception? DrainError; + } + + internal CommandExecutionFlow( + bool async, CommandList commands, TimeSpan? pendingTimeout = null) + : this(async, commands, pendingTimeout, null, null, null, null) + { } + + internal CommandExecutionFlow( + bool async, CommandList commands, TimeSpan? pendingTimeout, + Action? resultObserver, object? resultObserverState, + PgClientFlowObserver? lifecycleObserver, object? lifecycleState) + : base(supportsDeferredFlush: true) + { + if (commands.Count is 0) + ThrowHelper.ThrowArgumentException(nameof(commands), "A batch must contain at least one command."); + foreach (ref readonly var command in commands) + { + if (command.DescribeForPreparation || command.SuppressEnumeration) + ThrowHelper.ThrowArgumentException(nameof(commands), + "Preparation and suppressed commands require the general command flow."); + } + _commands = commands; + _pendingTimeout = pendingTimeout; + _resultObserver = resultObserver; + _resultObserverState = resultObserverState; + if (lifecycleObserver is not null) + SetObserver(lifecycleObserver, lifecycleState); + IsAsync = async; + if (!async) + _handoffEvent = new(false); + } + + internal override bool DefersSyncHandoff => true; + private protected override FlowHandoffEvent? HandoffEvent => _handoffEvent; + protected override bool EnableActivationTimeout => true; + protected override TimeSpan? PendingTimeout => _pendingTimeout; + + internal override void BindCallerToken(CancellationToken cancellationToken) + => _flowToken = cancellationToken; + internal override CancellationToken MigrationCancellationToken + => _flowToken; + + internal Enumerator GetEnumerator() + => new(this, default); + + internal Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + _flowToken = cancellationToken; + return new(this, cancellationToken); + } + + internal CommandResult? CurrentResult => _current; + internal bool IsResultReady => Volatile.Read(ref _phase) is PhaseResultReady; + internal int VisibleCommandCount => _commands.Count; + internal ValueTask MoveNextResultAsync(CancellationToken cancellationToken) + => MoveNextAsync(cancellationToken); + internal void DisposeResults() => Dispose(); + internal ValueTask DisposeResultsAsync() => DisposeAsync(); + + ColdState GetOrCreateColdState() + => Volatile.Read(ref _coldState) ?? + Interlocked.CompareExchange(ref _coldState, new(), null) ?? _coldState; + + bool IsClosed => Volatile.Read(ref _coldState)?.CloseException is not null; + bool IsCancelRequested => Volatile.Read(ref _coldState) is { CancelRequested: true }; + bool HasDecoder => HasSuccessfulActivation; + + protected override ValueTask ExecuteAuto(Context context) + { + _context = context; + ValueTask writeTask; + try + { + ref readonly var template = ref _commands.ItemRef(_commands.Count - 1); + var appendSync = !template.WithSync; + _readFlowRfq = appendSync; + // Caller cancellation never cancels wire I/O. The consumer observes the latched intent and + // drains its command to RFQ instead. + writeTask = IsAsync + ? _commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) + : WriteCommandsResumable(context, appendSync); + // Observe synchronous faults here; pending writes remain the framework-owned trailing task. + if (writeTask.IsCompleted) + writeTask.GetAwaiter().GetResult(); + } + catch (Exception ex) + { + // The framework recovers the wire from this throw. Only the consumer needs the fault. + FaultReady(ex); + throw; + } + + // Activation may precede or follow execution. Bridging it here guarantees a consumer resumes + // against a published context, and delivers an activation fault to the pipeline task when no + // consumer ever arrives. + var activation = context.GetDecoderAsync().ConfigureAwait(false); + if (activation.IsCompleted) + OnActivationSettled(onExecutorStrand: true); + else + activation.UnsafeOnCompleted(static state => ((CommandExecutionFlow)state!).OnActivationSettled(onExecutorStrand: false), this); + return new(new FlowTasks(writeTask, new ValueTask(this, _pipelineTaskSource.Version))); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + ValueTask WriteCommandsResumable(Context context, bool appendSync) + { + var encoder = context.GetEncoder(); + ValueTask writeTask; + using (encoder.BeginResumableWriteScope()) + writeTask = _commands.WriteCommandsResumable(encoder, appendSync); + return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); + } + + // Runs on the executor strand when activation already settled, else on the activation dispatch. + // The executor strand never runs consumer code. An activation dispatch is a detached work item + // whose only remaining work is this wake, so the consumer may continue on it directly. + void OnActivationSettled(bool onExecutorStrand) + { + try + { + _ = _context.GetDecoderAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + } + catch (Exception fault) + { + // Preserve the activation's close, timeout, or cancellation identity. The pipeline task + // faults first so a consumer woken by the ready source never races its retirement. + CompletePipelineTask(fault, runContinuationsAsynchronously: true); + FaultReady(fault); + return; + } + + if (IsCancelRequested) + RequestBackendCancellation(); + if (!CompleteReady(null, runContinuationsAsynchronously: onExecutorStrand)) + { + // Teardown released the consumer while this flow waited for its turn. Nothing reads the + // response, the closing wire owns it. + CompletePipelineTask(null, runContinuationsAsynchronously: true); + return; + } + // A cancel latched before activation may have released its caller already. The response + // still has to reach RFQ, so drain it unless a consumer already owns the decoder. + if (IsCancelRequested) + TryTakeOverDrain(); + } + + void FaultReady(Exception exception) + { + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + CompleteReady(exception, runContinuationsAsynchronously: true); + } + + bool CompleteReady(Exception? exception, bool runContinuationsAsynchronously) + { + if (Interlocked.CompareExchange(ref _readyCompletion, 1, 0) != 0) + return false; + if (exception is null) + _readySource.SetResult(true, runContinuationsAsynchronously); + else + _readySource.SetException(exception, runContinuationsAsynchronously); + return true; + } + + void CompletePipelineTask(Exception? exception, bool runContinuationsAsynchronously = false) + { + if (Interlocked.Exchange(ref _phase, PhaseCompleted) is PhaseCompleted) + return; + if (exception is null) + _pipelineTaskSource.SetResult(true, runContinuationsAsynchronously); + else + _pipelineTaskSource.SetException(exception, runContinuationsAsynchronously); + } + + void EnsureSyncHandoff() + { + if (IsAsyncAtDispatch) + ThrowHelper.ThrowInvalidOperation( + "Synchronous result consumption requires a flow initialized for synchronous execution."); + if (_syncHandoffClaimed) + return; + WaitForSyncHandoff(); + _syncHandoffClaimed = true; + } + + bool MoveNext() + { + EnsureSyncHandoff(); + while (true) + { + var phase = Volatile.Read(ref _phase); + switch (phase) + { + case PhaseInitial: + if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseInitial) != PhaseInitial) + continue; + _commandIndex = 0; + return First(); + case PhaseResultReady: + if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) != PhaseResultReady) + continue; + return NextBatch(); + case PhaseReading: + ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow."); + return false; + case PhaseDraining: + WaitForCompleteSynchronously(); + throw Volatile.Read(ref _coldState)?.TerminalException + ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); + default: + if (Volatile.Read(ref _coldState)?.TerminalException is { } terminal) + ExceptionDispatchInfo.Throw(terminal); + return false; + } + } + } + + bool First() + { + try + { + WaitForReadySynchronously(); + Debug.Assert(!_consumerDetached); + RegisterCancellation(default); + return PublishSynchronousResult(ReadResult(_commandIndex)); + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + } + + bool NextBatch() + { + try + { + RegisterCancellation(default); + var result = _current!; + var completeError = CompleteCurrentResult(); + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + SkipDiscardedCommands(); + + if (++_commandIndex < _commands.Count) + return PublishSynchronousResult(ReadResult(_commandIndex)); + + CompleteBatch(result); + _consumerObservedCompletion = true; + return false; + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + } + + bool PublishSynchronousResult(CommandResult result) + { + _current = result; + Interlocked.Exchange(ref _phase, PhaseResultReady); + var context = _context; + if (!IsClosed && context.StoppingToken.IsCancellationRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, + context.FlowTerminationException, null); + if (!IsCancelRequested && !IsClosed) + return true; + + if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + Drain(); + else + WaitForCompleteSynchronously(); + throw Volatile.Read(ref _coldState)?.TerminalException + ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); + } + + void WaitForReadySynchronously() + { + var ready = new ValueTask(this, _readySource.Version); + if (ready.IsCompleted) + _ = ready.GetAwaiter().GetResult(); + else + _ = ready.AsTask().GetAwaiter().GetResult(); + } + + ValueTask MoveNextAsync(CancellationToken cancellationToken) + { + if (!IsAsyncAtDispatch) + return ValueTask.FromException(ThrowHelper.ThrowInvalidOperation( + "Asynchronous result consumption requires a flow initialized for asynchronous execution.")); + while (true) + { + var phase = Volatile.Read(ref _phase); + switch (phase) + { + case PhaseInitial: + if (cancellationToken.IsCancellationRequested) + return CancelBeforeRead(cancellationToken); + if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseInitial) != PhaseInitial) + continue; + _commandIndex = 0; + return FirstAsync(cancellationToken); + case PhaseResultReady: + if (cancellationToken.IsCancellationRequested) + return CancelBeforeRead(cancellationToken); + if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) != PhaseResultReady) + continue; + return NextBatchAsync(cancellationToken); + case PhaseReading: + return ValueTask.FromException( + ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow.")); + case PhaseDraining: + return AwaitTakeoverAsync(); + default: + return Volatile.Read(ref _coldState)?.TerminalException is { } terminal + ? ValueTask.FromException(terminal) + : new(false); + } + } + } + + // A pre-cancelled token releases the caller immediately. The wire still drains to RFQ. + ValueTask CancelBeforeRead(CancellationToken cancellationToken) + { + RequestCancel(cancellationToken); + return ValueTask.FromException(new OperationCanceledException(cancellationToken)); + } + + // The consumer parks behind a takeover drain and receives the outcome that caused it. + async ValueTask AwaitTakeoverAsync() + { + await WaitForCompletionAsync().ConfigureAwait(false); + throw Volatile.Read(ref _coldState)?.TerminalException ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask FirstAsync(CancellationToken cancellationToken) + { + Exception? deliver; + try + { + await new ValueTask(this, _readySource.Version).ConfigureAwait(false); + Debug.Assert(!_consumerDetached); + RegisterCancellation(cancellationToken); + CommandResult result; + if (!_commands.ItemRef(_commandIndex).DescribeOnly + && _commands.ItemRef(_commandIndex).Descriptor + is { IsPrepared: true, PreparedRowDescription: not null }) + { + var decoder = _context.Decoder; + if (_context.IsProtocolClosed) + throw _context.FlowTerminationException; + decoder.UseReadTimeout(_commands.ItemRef(_commandIndex).Timeout); + PgError? error; + if (!decoder.TryMoveNext()) + { + if (!await decoder.MoveNextAsync().ConfigureAwait(false)) + decoder.ThrowUnexpectedEof(); + } + if (decoder.Current.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) + { + error = bindError; + } + else + { + if (!decoder.TryMoveNext()) + { + if (!await decoder.MoveNextAsync().ConfigureAwait(false)) + decoder.ThrowUnexpectedEof(); + } + decoder.Current.DebugEnsureExpected( + PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); + error = null; + } + result = InitializeResult(_commandIndex, error, null); + } + else + { + result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + } + _current = result; + // Publish the idle state, then recheck the latches. A latch that landed between the read + // and this publication found no idle owner to take over, so this frame must act on it. + Interlocked.Exchange(ref _phase, PhaseResultReady); + // Graceful stopping faults a result that arrives after the close began, as the ordinary + // flow does at each result boundary. Latch it so the drain delivers that close. + var context = _context; + if (!IsClosed && context.StoppingToken.IsCancellationRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, + context.FlowTerminationException, null); + if (!IsCancelRequested && !IsClosed) + return true; + if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + { + await DrainAsync().ConfigureAwait(false); + } + else + { + // The latching side took the decoder first. Park behind its drain. + await WaitForCompletionAsync().ConfigureAwait(false); + } + deliver = Volatile.Read(ref _coldState)?.TerminalException; + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + throw deliver ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask NextBatchAsync(CancellationToken cancellationToken) + { + try + { + RegisterCancellation(cancellationToken); + var result = _current!; + var resultEnumerator = _context.GetProtocolStatic() + .ResultMessageEnumerator; + await resultEnumerator.DisposeAsync().ConfigureAwait(false); + var completeError = resultEnumerator.CompleteError; + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + await SkipDiscardedCommandsAsync().ConfigureAwait(false); + + if (++_commandIndex < _commands.Count) + { + result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + _current = result; + Interlocked.Exchange(ref _phase, PhaseResultReady); + var context = _context; + if (!IsClosed && context.StoppingToken.IsCancellationRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, + context.FlowTerminationException, null); + if (!IsCancelRequested && !IsClosed) + return true; + + if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + await DrainAsync().ConfigureAwait(false); + else + await WaitForCompletionAsync().ConfigureAwait(false); + throw Volatile.Read(ref _coldState)?.TerminalException + ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); + } + + await CompleteBatchAsync(result).ConfigureAwait(false); + _consumerObservedCompletion = true; + return false; + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + } + + // Reads through the command's execute prelude and initializes the protocol-static result. + async ValueTask ReadResultAsync(int commandIndex) + { + var context = _context; + var decoder = context.Decoder; + // After close, a fresh command must not consume bytes left by its predecessor. + if (context.IsProtocolClosed) + throw context.FlowTerminationException; + PgError? error; + RowDescription? requestedRowDescription; + var describeOnly = _commands.ItemRef(commandIndex).DescribeOnly; + var hasPreparedDescription = _commands.ItemRef(commandIndex).Descriptor + is { IsPrepared: true, PreparedRowDescription: not null }; + decoder.UseReadTimeout(_commands.ItemRef(commandIndex).Timeout); + if (hasPreparedDescription && !describeOnly) + { + // Prepared commands with a known description have the compact BindComplete -> + // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes this + // frame rather than a nested parser coroutine. + if (!decoder.TryMoveNext()) + { + if (!await decoder.MoveNextAsync().ConfigureAwait(false)) + decoder.ThrowUnexpectedEof(); + } + var message = decoder.Current; + if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) + { + error = bindError; + } + else + { + if (!decoder.TryMoveNext()) + { + if (!await decoder.MoveNextAsync().ConfigureAwait(false)) + decoder.ThrowUnexpectedEof(); + } + decoder.Current.DebugEnsureExpected(PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); + error = null; + } + requestedRowDescription = null; + } + else + { + (error, requestedRowDescription) = await _commands.ItemRef(commandIndex) + .ReadUntilExecuteAsync(decoder, context.GetProtocolStatic().RowDescription) + .ConfigureAwait(false); + } + return InitializeResult(commandIndex, error, requestedRowDescription); + } + + CommandResult ReadResult(int commandIndex) + { + var context = _context; + var decoder = context.Decoder; + if (context.IsProtocolClosed) + throw context.FlowTerminationException; + ref readonly var command = ref _commands.ItemRef(commandIndex); + decoder.UseReadTimeout(command.Timeout); + var (error, requestedRowDescription) = command + .ReadUntilExecute(decoder, context.GetProtocolStatic().RowDescription); + return InitializeResult(commandIndex, error, requestedRowDescription); + } + + CommandResult InitializeResult( + int commandIndex, PgError? error, RowDescription? requestedRowDescription) + { + var context = _context; + ref readonly var readState = ref context.GetProtocolStatic(); + ref readonly var command = ref _commands.ItemRef(commandIndex); + readState.ResultMessageEnumerator.Initialize(command, context.Decoder); + var result = readState.CommandResult; + var descriptor = command.Descriptor; + // A named unprepared statement that parsed becomes a prepared descriptor. + if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault + && (error is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) + { + descriptor = CommandDescriptor.CreatePrepared(descriptor.CommandName, descriptor.ParameterTypes, + requestedRowDescription?.Preserve()); + } + result.Initialize(this, commandIndex, descriptor, requestedRowDescription, + !command.DescribeOnly, command.IsSimple(), error); + _resultObserver?.Invoke(result, _resultObserverState); + return result; + } + + async ValueTask<(PgError Error, TransactionStatus TransactionStatus)?> CompleteCurrentResultAsync() + { + var enumerator = _context.GetProtocolStatic().ResultMessageEnumerator; + await enumerator.DisposeAsync().ConfigureAwait(false); + return enumerator.CompleteError; + } + + (PgError Error, TransactionStatus TransactionStatus)? CompleteCurrentResult() + { + var enumerator = _context.GetProtocolStatic().ResultMessageEnumerator; + enumerator.Dispose(); + return enumerator.CompleteError; + } + + async ValueTask SkipDiscardedCommandsAsync() + { + while (++_commandIndex < _commands.Count && !_commands[_commandIndex].WithSync) { } + await ReadRfqAsync().ConfigureAwait(false); + if (_commandIndex == _commands.Count) + _readFlowRfq = false; + } + + void SkipDiscardedCommands() + { + while (++_commandIndex < _commands.Count && !_commands[_commandIndex].WithSync) { } + ReadRfq(); + if (_commandIndex == _commands.Count) + _readFlowRfq = false; + } + + async ValueTask ReadRfqAsync() + { + var message = await _context.Decoder.GetNextAsync().ConfigureAwait(false); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } + + void ReadRfq() + { + var message = _context.Decoder.GetNext(); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } + + async ValueTask CompleteBatchAsync(CommandResult result) + { + if (_readFlowRfq) + await ReadRfqAsync().ConfigureAwait(false); + await DisposeRegistrationsAsync().ConfigureAwait(false); + Finish(result); + } + + void CompleteBatch(CommandResult result) + { + if (_readFlowRfq) + ReadRfq(); + DisposeRegistrations(); + Finish(result); + } + + // The wire is at this command's RFQ. Release the shared read objects, record the outcome the + // consumer must observe, then complete the pipeline task. + void Finish(CommandResult result) + { + if (result.Error is { } error && _consumerDetached && !IsOwnCancellation(error)) + GetOrCreateColdState().DrainError = PgErrorException.Create(error); + _context.GetProtocolStatic().Reset(); + _current = null; + if (IsCancelRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, + new OperationCanceledException(_coldState!.DeliverToken), null); + else if (Volatile.Read(ref _coldState)?.CloseException is { } close) + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, close, null); + CompletePipelineTask(null); + } + + bool IsOwnCancellation(PgError error) + => IsCancelRequested && error.SqlState == PgErrorCodes.QueryCanceled; + + // The owning frame failed. The read error is the pipeline task's failure, which the framework + // recovers or drains. Later consumer calls replay it. + void FaultFromOwner(Exception exception) + { + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + if (Volatile.Read(ref _phase) == PhaseCompleted) + return; + DisposeRegistrations(); + _current = null; + if (HasDecoder) + _context.GetProtocolStatic().Reset(); + CompletePipelineTask(exception); + } + + // Takes an idle decoder for a drain. Returns false when a frame already owns it, which will observe + // the latch itself, or when the flow reached its terminal. + bool TryTakeOverDrain() + { + while (true) + { + var phase = Volatile.Read(ref _phase); + if (phase is PhaseInitial && !HasDecoder) + return false; + if (phase is not (PhaseInitial or PhaseResultReady)) + return false; + if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) + continue; + _consumerDetached = true; + ThreadPool.UnsafeQueueUserWorkItem(static state => _ = ((CommandExecutionFlow)state!).DrainAsync(), this); + return true; + } + } + + // Autonomous drain. Owns the decoder until the pipeline task completes. Never throws. + async ValueTask DrainAsync() + { + try + { + var result = _current; + if (result is null) + { + await new ValueTask(this, _readySource.Version).ConfigureAwait(false); + _commandIndex = 0; + result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + } + + while (true) + { + var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + await SkipDiscardedCommandsAsync().ConfigureAwait(false); + if (++_commandIndex >= _commands.Count) + break; + result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + } + await CompleteBatchAsync(result).ConfigureAwait(false); + } + catch (Exception ex) + { + FaultFromOwner(ex); + } + } + + void Drain() + { + try + { + var result = _current; + if (result is null) + { + // Synchronous disposal before any read. The activation bridge normally completed long + // ago, so this bridge is rarely more than a status check. + new ValueTask(this, _readySource.Version).AsTask().GetAwaiter().GetResult(); + _commandIndex = 0; + result = ReadResult(_commandIndex); + } + + while (true) + { + var completeError = CompleteCurrentResult(); + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + SkipDiscardedCommands(); + if (++_commandIndex >= _commands.Count) + break; + result = ReadResult(_commandIndex); + } + CompleteBatch(result); + } + catch (Exception ex) + { + FaultFromOwner(ex); + } + } + + ValueTask DisposeAsync() + { + while (true) + { + var phase = Volatile.Read(ref _phase); + switch (phase) + { + case PhaseInitial: + case PhaseResultReady: + if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) + continue; + _consumerDetached = true; + if (_current is { IsComplete: false } + || _commandIndex + 1 < _commands.Count) + RequestCancel(default); + return WaitForDrainOnDispose ? DisposeDrainAsync() : FireAndForgetDrain(); + case PhaseReading: + return ValueTask.FromException( + ThrowHelper.ThrowInvalidOperation("Cannot dispose the flow while a read is in progress.")); + default: + return !WaitForDrainOnDispose || _consumerObservedCompletion + ? default + : DisposeCompletedAsync(); + } + } + } + + // Drains on the disposer's frame, then waits for framework release so a drain error can surface. + async ValueTask DisposeDrainAsync() + { + await DrainAsync().ConfigureAwait(false); + await DisposeCompletedAsync().ConfigureAwait(false); + } + + ValueTask FireAndForgetDrain() + { + _ = DrainAsync(); + return default; + } + + async ValueTask DisposeCompletedAsync() + { + await WaitForCompletionAsync().ConfigureAwait(false); + if (Volatile.Read(ref _coldState)?.DrainError is { } drainError) + throw drainError; + } + + // Flow completion is independent of errors accumulated while draining. A close is a clean + // terminal for a disposing consumer. + async ValueTask WaitForCompletionAsync() + { + try + { + await WaitForComplete().ConfigureAwait(false); + } + catch (PgClientClosedException) + { + } + } + + void Dispose() + { + if (!IsAsyncAtDispatch) + EnsureSyncHandoff(); + while (true) + { + var phase = Volatile.Read(ref _phase); + switch (phase) + { + case PhaseInitial: + case PhaseResultReady: + if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) + continue; + _consumerDetached = true; + if (_current is { IsComplete: false } + || _commandIndex + 1 < _commands.Count) + RequestCancel(default); + Drain(); + if (WaitForDrainOnDispose) + DisposeCompleted(); + return; + case PhaseReading: + ThrowHelper.ThrowInvalidOperation("Cannot dispose the flow while a read is in progress."); + return; + default: + if (WaitForDrainOnDispose && !_consumerObservedCompletion) + DisposeCompleted(); + return; + } + } + } + + void DisposeCompleted() + { + try + { + WaitForCompleteSynchronously(); + } + catch (PgClientClosedException) + { + } + if (Volatile.Read(ref _coldState)?.DrainError is { } drainError) + throw drainError; + } + + // When true, disposal waits for the drain to reach RFQ and for framework release. Otherwise it + // returns while the drain continues autonomously. + internal bool WaitForDrainOnDispose { get; set; } = true; + + void RegisterCancellation(CancellationToken callerToken) + { + if (callerToken == _flowToken && + (_flowRegistration != default || !callerToken.CanBeCanceled)) + return; + var registration = _flowRegistration; + _flowRegistration = default; + registration.Dispose(); + _flowToken = callerToken; + if (callerToken.CanBeCanceled) + _flowRegistration = callerToken.UnsafeRegister(static (state, token) + => ((CommandExecutionFlow)state!).RequestCancel(token), this); + } + + ValueTask DisposeRegistrationsAsync() + { + if (_flowRegistration == default) + return default; + var flowRegistration = _flowRegistration; + _flowRegistration = default; + return flowRegistration.DisposeAsync(); + } + + void DisposeRegistrations() + { + var flowRegistration = _flowRegistration; + _flowRegistration = default; + flowRegistration.Dispose(); + } + + // Cancellation only latches intent and requests a backend cancel. The frame owning the decoder + // delivers it after the wire is back at RFQ. An idle flow drains autonomously first. + void RequestCancel(CancellationToken token) + { + if (Volatile.Read(ref _phase) == PhaseCompleted) + return; + var cancellation = GetOrCreateColdState(); + cancellation.DeliverToken = token; + Interlocked.Exchange(ref cancellation.CancelRequested, true); + if (HasDecoder) + RequestBackendCancellation(); + TryTakeOverDrain(); + } + + internal Task CancelAsync() + { + RequestCancel(default); + return WaitForComplete().AsTask(); + } + + void RequestBackendCancellation() + => _context.RequestBackendCancellation(this, CancellationWindow, BackendCancellationTiming.AfterGrace); + + // Finish and FaultFromOwner reset the shared read objects before the pipeline task completes, and + // Current is null once the consumer observed the terminal, so nothing outlives the flow. + internal override bool ResetsSharedReadStateBeforeRelease => true; + + // Graceful stop. An unactivated flow releases its consumer, the closing wire owns its response. + // An idle activated flow drains itself to RFQ so the pipeline can complete. + protected override void OnStopping(Exception exception) + { + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); + if (CompleteReady(exception, runContinuationsAsynchronously: true)) + { + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + return; + } + TryTakeOverDrain(); + } + + // Forceful abort. No frame can read a dead wire, so an idle owner faults the pipeline task + // directly. A frame in flight fails on its own read. + protected override void OnAbort(Exception exception) + { + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); + if (CompleteReady(exception, runContinuationsAsynchronously: true)) + { + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + return; + } + while (true) + { + var phase = Volatile.Read(ref _phase); + if (phase is not (PhaseInitial or PhaseResultReady)) + return; + if (Interlocked.CompareExchange(ref _phase, PhaseCompleted, phase) != phase) + continue; + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + _pipelineTaskSource.SetException(exception, runContinuationsAsynchronously: true); + return; + } + } + + internal override void Fail(Exception exception) + { + // A result callback failed on the frame that owns the decoder. Its throw propagates there. + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + } + + protected override void OnReleasing(Exception? exception) + { + DisposeRegistrations(); + _commands.Return(); + } + + protected override void OnDiscarded() + { + GetObserver(out var observerState)?.OnCompleting(this, null, observerState); + _commands.Return(); + } + + protected override void OnReset() + { + _phase = PhaseInitial; + _commandIndex = -1; + _context = default; + _current = null; + _readFlowRfq = false; + _consumerDetached = false; + _consumerObservedCompletion = false; + _readySource.Reset(); + _pipelineTaskSource.Reset(); + _readyCompletion = 0; + _coldState = null; + _syncHandoffClaimed = false; + _handoffEvent?.ResetInteraction(); + WaitForDrainOnDispose = true; + } + + bool IValueTaskSource.GetResult(short token) => _readySource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _readySource.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _readySource.OnCompleted(continuation, state, token, flags); + + void IValueTaskSource.GetResult(short token) => _pipelineTaskSource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _pipelineTaskSource.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _pipelineTaskSource.OnCompleted(continuation, state, token, flags); + + public readonly struct Enumerator : IAsyncEnumerator, IDisposable + { + readonly CommandExecutionFlow? _flow; + readonly CancellationToken _cancellationToken; + + public Enumerator(CommandExecutionFlow flow) + : this(flow, default) + { } + + internal Enumerator(CommandExecutionFlow flow, CancellationToken cancellationToken) + { + _flow = flow; + _cancellationToken = cancellationToken; + } + + public Enumerator GetAsyncEnumerator() => this; + + public Enumerator GetEnumerator() => this; + + public bool MoveNext() => _flow?.MoveNext() ?? false; + + public ValueTask MoveNextAsync() => MoveNextAsync(_cancellationToken); + + public ValueTask MoveNextAsync(CancellationToken cancellationToken) + => _flow is null ? new(false) : _flow.MoveNextAsync(cancellationToken); + + public CommandResult Current => _flow?._current ?? default!; + + public ValueTask DisposeAsync() => _flow is null ? default : _flow.DisposeAsync(); + + public void Dispose() => _flow?.Dispose(); + } +} diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 259ee9c..d4c6406 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -488,6 +488,10 @@ protected virtual void OnReleasing(Exception? exception) {} protected virtual void OnDiscarded() {} protected virtual void OnReset() {} + private protected bool HasSuccessfulActivation + => _activationTaskSource.GetStatus(_activationTaskSource.Version) + is ValueTaskSourceStatus.Succeeded; + // The per-flow handoff rendezvous primitive for the (wait-list-free) sync source handoff: non-null only // for a flow that needs a caller takeover (a sync CommandFlow with a parked caller). The source signals // it when it dequeues-and-holds the flow for that caller (OnExecutorSuspended), and the caller parks on @@ -528,6 +532,7 @@ protected readonly struct Context readonly ExecutionControl _executionControl; internal Context(ExecutionControl executionControl) => _executionControl = executionControl; + internal PgDecoder Decoder => _executionControl.Decoder; /// Graceful drain signal. Poll at handoff/coordination boundaries (per-CommandResult for /// CommandFlow) to switch to drain mode. I/O keeps running so the wire reaches a clean @@ -741,6 +746,7 @@ public void UnsafeOnCompleted(Action continuation, object? state) internal readonly struct ExecutionControl(PgClientFlow flow, PgClientProtocol.Control control) { internal PgClientFlow Flow => flow; + internal PgDecoder Decoder => control.Decoder; public bool SupportsDeferredFlush => flow is { _supportsDeferredFlush: true, _isAsyncAtDispatch: true }; public bool StallsPipeline => !SupportsDeferredFlush; From 6e4f0cf84e0e604553fa8881c3633bd7e6f79079 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 00:16:56 +0200 Subject: [PATCH 083/136] Run the command flow suite against the replacement --- Slon.Tests/CommandFlowImplementation.cs | 5 + Slon.Tests/FlowBindingProbe.cs | 3 +- .../Pg/CommandResultEnumerationTests.cs | 7 + Slon.Tests/Pg/ExclusiveAccessFlowTests.cs | 11 +- Slon.Tests/Slon.Tests.csproj | 2 + .../Pg/Protocol/Flows/CommandExecutionFlow.cs | 138 ++++++++++++++---- test-command-flows.sh | 9 ++ 7 files changed, 145 insertions(+), 30 deletions(-) create mode 100644 Slon.Tests/CommandFlowImplementation.cs create mode 100755 test-command-flows.sh diff --git a/Slon.Tests/CommandFlowImplementation.cs b/Slon.Tests/CommandFlowImplementation.cs new file mode 100644 index 0000000..b71bc62 --- /dev/null +++ b/Slon.Tests/CommandFlowImplementation.cs @@ -0,0 +1,5 @@ +#if COMMAND_FLOW_NEXT +global using CommandFlow = Slon.Pg.Protocol.Flows.CommandExecutionFlow; +global using CommandFlowObserver = Slon.Pg.Protocol.Flows.CommandExecutionFlowObserver; +global using CommandFlowOptions = Slon.Pg.Protocol.Flows.CommandExecutionFlowOptions; +#endif diff --git a/Slon.Tests/FlowBindingProbe.cs b/Slon.Tests/FlowBindingProbe.cs index 00a0289..b011a6d 100644 --- a/Slon.Tests/FlowBindingProbe.cs +++ b/Slon.Tests/FlowBindingProbe.cs @@ -1,6 +1,7 @@ using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; using Slon.Pg; +using LegacyCommandFlow = Slon.Pg.Protocol.Flows.CommandFlow; namespace Slon.Tests; @@ -9,7 +10,7 @@ sealed class BindingProbeContext(string name) : PgClientFlowBindingContext internal string Name { get; } = name; } -sealed class BindingProbeFlow(bool fail = false) : CommandFlow(async: true, []) +sealed class BindingProbeFlow(bool fail = false) : LegacyCommandFlow(async: true, []) { internal int BindCount { get; private set; } internal string? ContextName { get; private set; } diff --git a/Slon.Tests/Pg/CommandResultEnumerationTests.cs b/Slon.Tests/Pg/CommandResultEnumerationTests.cs index c712c6c..3dd25ec 100644 --- a/Slon.Tests/Pg/CommandResultEnumerationTests.cs +++ b/Slon.Tests/Pg/CommandResultEnumerationTests.cs @@ -241,7 +241,12 @@ public async Task ErrorWithoutSync_ResumesAfterInternalSync(bool async) public async Task Reset_ClearsEnumerationCompleted_ForNextTenure() { await using var protocol = await PgTestPool.NewIsolatedAsync(); +#if COMMAND_FLOW_NEXT + var flow = new CommandFlow( + async: true, enableActivationTimeout: false, Command.Create("select 1")); +#else var flow = new ResettableCommandFlow(async: true, Command.Create("select 1")); +#endif for (var tenure = 0; tenure < 2; tenure++) { if (tenure > 0) @@ -259,9 +264,11 @@ public async Task Reset_ClearsEnumerationCompleted_ForNextTenure() } // Pooling a timeout-armed flow is refused by Reset. Opt out so the reset path itself is testable. +#if !COMMAND_FLOW_NEXT sealed class ResettableCommandFlow(bool async, params ReadOnlySpan commands) : CommandFlow(async, commands) { protected override bool EnableActivationTimeout => false; } +#endif } diff --git a/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs b/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs index 184353a..e4797c6 100644 --- a/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs +++ b/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs @@ -25,7 +25,7 @@ public async Task BindingFailure_FaultsFlowWithoutCondemningWire() protocol.SetFlowBindingContext(new BindingProbeContext("wire")); var flow = protocol.Queue(new BindingProbeFlow(fail: true)); - await Assert.ThrowsExactlyAsync(async () => await DrainAsync(flow)); + await Assert.ThrowsExactlyAsync(async () => await DrainBindingProbeAsync(flow)); Assert.AreEqual(1, flow.BindCount); await DrainAsync(protocol.Queue(new CommandFlow(async: true, Command.Create("select 1")))); @@ -37,6 +37,13 @@ static async Task DrainAsync(CommandFlow flow) await e.DisposeAsync(); } + static async Task DrainBindingProbeAsync(BindingProbeFlow flow) + { + var e = flow.GetAsyncEnumerator(); + while (await e.MoveNextAsync()) { } + await e.DisposeAsync(); + } + [TestMethod] public async Task Scope_RoundTrip_RunsCommandOnInnerPipeline() { @@ -86,7 +93,7 @@ public async Task LongRunningScope_RejectsOuterAdmissionUntilReleased() Assert.IsTrue(protocol.IsSchedulable); Assert.IsTrue(protocol.TryQueue(bindProbe)); - await DrainAsync(bindProbe); + await DrainBindingProbeAsync(bindProbe); Assert.AreEqual(1, bindProbe.BindCount, "the portable flow must bind exactly once when its later placement reaches dispatch"); } diff --git a/Slon.Tests/Slon.Tests.csproj b/Slon.Tests/Slon.Tests.csproj index 5273122..1df14ac 100644 --- a/Slon.Tests/Slon.Tests.csproj +++ b/Slon.Tests/Slon.Tests.csproj @@ -8,6 +8,8 @@ false true $(NoWarn);SLONPG001;SLONPOOL001 + Legacy + $(DefineConstants);COMMAND_FLOW_NEXT diff --git a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs index d3a441e..908aff3 100644 --- a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs @@ -6,6 +6,22 @@ namespace Slon.Pg.Protocol.Flows; +internal abstract class CommandExecutionFlowObserver : PgClientFlowObserver +{ + protected internal virtual void OnStarted(CommandExecutionFlow flow, object? state) { } + protected internal virtual void OnCommandResult( + CommandExecutionFlow flow, CommandResult result, object? state) { } + protected internal virtual void OnDrainStarted(CommandExecutionFlow flow, object? state) { } +} + +internal readonly struct CommandExecutionFlowOptions +{ + public CommandExecutionFlowObserver? Observer { get; init; } + public object? ObserverState { get; init; } + public CommandList Commands { get; init; } + public TimeSpan? PendingTimeout { get; init; } +} + // Replacement-flow prototype: one consumer-owned decoder lifecycle for synchronous and asynchronous // execution, with the general multi-command result shape. Kept internal until it covers the complete // CommandFlow contract; the eventual single-command specialization will remain a separate sealed type. @@ -20,10 +36,10 @@ internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource? _resultObserver; - readonly object? _resultObserverState; + CommandList _commands; + TimeSpan? _pendingTimeout; + CommandExecutionFlowObserver? _commandObserver; + object? _commandObserverState; int _commandIndex = -1; Context _context; CommandResult? _current; @@ -45,6 +61,8 @@ internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource commands) + : this(async) + => Initialize(async, commands); internal CommandExecutionFlow( - bool async, CommandList commands, TimeSpan? pendingTimeout, - Action? resultObserver, object? resultObserverState, - PgClientFlowObserver? lifecycleObserver, object? lifecycleState) - : base(supportsDeferredFlush: true) + bool async, bool enableActivationTimeout, params ReadOnlySpan commands) + : this(async, commands) + => _enableActivationTimeout = enableActivationTimeout; + + internal CommandExecutionFlow(bool async, CommandList commands, TimeSpan? pendingTimeout = null) + : this(async, pendingTimeout) + => Initialize(async, new CommandExecutionFlowOptions + { + Commands = commands, + PendingTimeout = pendingTimeout + }); + + internal CommandExecutionFlow(bool async, in CommandExecutionFlowOptions options) + : this(async, options.PendingTimeout) + => Initialize(async, options); + + internal CommandExecutionFlow Initialize(bool async, params ReadOnlySpan commands) + => Initialize(async, new CommandExecutionFlowOptions { Commands = new(commands) }); + + internal CommandExecutionFlow Initialize(bool async, in CommandExecutionFlowOptions options) { + IsAsync = async; + if (!async) + _handoffEvent ??= new(false); + var commands = options.Commands; if (commands.Count is 0) - ThrowHelper.ThrowArgumentException(nameof(commands), "A batch must contain at least one command."); + return this; foreach (ref readonly var command in commands) { if (command.DescribeForPreparation || command.SuppressEnumeration) - ThrowHelper.ThrowArgumentException(nameof(commands), - "Preparation and suppressed commands require the general command flow."); + ThrowHelper.ThrowArgumentException(nameof(options), + "Preparation and suppressed commands are not implemented by the replacement flow yet."); } _commands = commands; - _pendingTimeout = pendingTimeout; - _resultObserver = resultObserver; - _resultObserverState = resultObserverState; - if (lifecycleObserver is not null) - SetObserver(lifecycleObserver, lifecycleState); - IsAsync = async; - if (!async) - _handoffEvent = new(false); + _pendingTimeout = options.PendingTimeout; + _commandObserver = options.Observer; + _commandObserverState = options.ObserverState; + if (options.Observer is { } observer) + { + SetObserver(observer, options.ObserverState); + observer.OnStarted(this, options.ObserverState); + } + return this; } internal override bool DefersSyncHandoff => true; private protected override FlowHandoffEvent? HandoffEvent => _handoffEvent; - protected override bool EnableActivationTimeout => true; + protected override bool EnableActivationTimeout => _enableActivationTimeout; protected override TimeSpan? PendingTimeout => _pendingTimeout; internal override void BindCallerToken(CancellationToken cancellationToken) @@ -97,10 +144,10 @@ internal override void BindCallerToken(CancellationToken cancellationToken) internal override CancellationToken MigrationCancellationToken => _flowToken; - internal Enumerator GetEnumerator() + public Enumerator GetEnumerator() => new(this, default); - internal Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { _flowToken = cancellationToken; return new(this, cancellationToken); @@ -114,6 +161,33 @@ internal ValueTask MoveNextResultAsync(CancellationToken cancellationToken internal void DisposeResults() => Dispose(); internal ValueTask DisposeResultsAsync() => DisposeAsync(); + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + internal async ValueTask ConsumeNonQueryAsync( + CancellationToken cancellationToken = default) + { + var recordsAffected = -1L; + var results = GetAsyncEnumerator(cancellationToken); + try + { + while (await results.MoveNextAsync().ConfigureAwait(false)) + { + var result = results.Current; + await result.CompleteAsync().ConfigureAwait(false); + result.GetCommandComplete(); + if (result.RecordsAffected is { } affected and >= 0) + recordsAffected = recordsAffected < 0 + ? affected + : checked(recordsAffected + affected); + } + return recordsAffected; + } + finally + { + await results.DisposeAsync().ConfigureAwait(false); + } + } + ColdState GetOrCreateColdState() => Volatile.Read(ref _coldState) ?? Interlocked.CompareExchange(ref _coldState, new(), null) ?? _coldState; @@ -593,7 +667,7 @@ CommandResult InitializeResult( } result.Initialize(this, commandIndex, descriptor, requestedRowDescription, !command.DescribeOnly, command.IsSimple(), error); - _resultObserver?.Invoke(result, _resultObserverState); + _commandObserver?.OnCommandResult(this, result, _commandObserverState); return result; } @@ -703,12 +777,19 @@ bool TryTakeOverDrain() return false; if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) continue; + NotifyDrainStarted(); _consumerDetached = true; ThreadPool.UnsafeQueueUserWorkItem(static state => _ = ((CommandExecutionFlow)state!).DrainAsync(), this); return true; } } + void NotifyDrainStarted() + { + if (Interlocked.Exchange(ref _drainStarted, 1) is 0) + _commandObserver?.OnDrainStarted(this, _commandObserverState); + } + // Autonomous drain. Owns the decoder until the pipeline task completes. Never throws. async ValueTask DrainAsync() { @@ -781,6 +862,7 @@ ValueTask DisposeAsync() case PhaseResultReady: if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) continue; + NotifyDrainStarted(); _consumerDetached = true; if (_current is { IsComplete: false } || _commandIndex + 1 < _commands.Count) @@ -843,6 +925,7 @@ void Dispose() case PhaseResultReady: if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) continue; + NotifyDrainStarted(); _consumerDetached = true; if (_current is { IsComplete: false } || _commandIndex + 1 < _commands.Count) @@ -1002,6 +1085,7 @@ protected override void OnReset() _readySource.Reset(); _pipelineTaskSource.Reset(); _readyCompletion = 0; + _drainStarted = 0; _coldState = null; _syncHandoffClaimed = false; _handoffEvent?.ResetInteraction(); diff --git a/test-command-flows.sh b/test-command-flows.sh new file mode 100755 index 0000000..750b398 --- /dev/null +++ b/test-command-flows.sh @@ -0,0 +1,9 @@ +#!/bin/zsh +set -eu + +repo=${0:A:h} + +dotnet test "$repo/Slon.Tests/Slon.Tests.csproj" -c Release \ + -p:CommandFlowImplementation=Legacy "$@" +dotnet test "$repo/Slon.Tests/Slon.Tests.csproj" -c Release \ + -p:CommandFlowImplementation=Next "$@" From 7d3e7aefb1c395ed6b7728064c4688dc1faca0ac Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 00:21:13 +0200 Subject: [PATCH 084/136] Classify legacy flow mode switching explicitly --- Slon.Tests/Pg/CommandResultEnumerationTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Slon.Tests/Pg/CommandResultEnumerationTests.cs b/Slon.Tests/Pg/CommandResultEnumerationTests.cs index 3dd25ec..39e54a3 100644 --- a/Slon.Tests/Pg/CommandResultEnumerationTests.cs +++ b/Slon.Tests/Pg/CommandResultEnumerationTests.cs @@ -128,6 +128,9 @@ public async Task DescribeOnlyErrorSurfacesWhenInspectingTheResult() } [ConnectionCreatingTestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("The replacement fixes the execution/consumption mode for a flow tenure; switching an async flow to synchronous driving is a legacy body-rendezvous behavior.")] +#endif public async Task AsyncFlow_CanSwitchToSynchronousResultAdvancement() { await using var protocol = await PgTestPool.NewIsolatedAsync(); From 616cddcf5d60b6df24094442ee4982a4d6fcec95 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 00:26:30 +0200 Subject: [PATCH 085/136] Support preparation and suppressed command results --- .../Pg/Protocol/Flows/CommandExecutionFlow.cs | 205 ++++++++++++------ 1 file changed, 142 insertions(+), 63 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs index 908aff3..8d476da 100644 --- a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs @@ -43,6 +43,7 @@ internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource FirstAsync(CancellationToken cancellationToken) await new ValueTask(this, _readySource.Version).ConfigureAwait(false); Debug.Assert(!_consumerDetached); RegisterCancellation(cancellationToken); - CommandResult result; - if (!_commands.ItemRef(_commandIndex).DescribeOnly - && _commands.ItemRef(_commandIndex).Descriptor - is { IsPrepared: true, PreparedRowDescription: not null }) - { - var decoder = _context.Decoder; - if (_context.IsProtocolClosed) - throw _context.FlowTerminationException; - decoder.UseReadTimeout(_commands.ItemRef(_commandIndex).Timeout); - PgError? error; - if (!decoder.TryMoveNext()) - { - if (!await decoder.MoveNextAsync().ConfigureAwait(false)) - decoder.ThrowUnexpectedEof(); - } - if (decoder.Current.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) - { - error = bindError; - } - else - { - if (!decoder.TryMoveNext()) - { - if (!await decoder.MoveNextAsync().ConfigureAwait(false)) - decoder.ThrowUnexpectedEof(); - } - decoder.Current.DebugEnsureExpected( - PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); - error = null; - } - result = InitializeResult(_commandIndex, error, null); - } - else - { - result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); - } + var result = await ReadNextPublishedResultAsync().ConfigureAwait(false); + if (result is null) + return false; _current = result; + _currentPublished = true; // Publish the idle state, then recheck the latches. A latch that landed between the read // and this publication found no idle owner to take over, so this frame must act on it. Interlocked.Exchange(ref _phase, PhaseResultReady); @@ -552,13 +518,17 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) .ResultMessageEnumerator; await resultEnumerator.DisposeAsync().ConfigureAwait(false); var completeError = resultEnumerator.CompleteError; + _currentPublished = false; if (completeError is { TransactionStatus: TransactionStatus.Unknown }) await SkipDiscardedCommandsAsync().ConfigureAwait(false); - if (++_commandIndex < _commands.Count) + _commandIndex++; + var next = await ReadNextPublishedResultAsync().ConfigureAwait(false); + if (next is not null) { - result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + result = next; _current = result; + _currentPublished = true; Interlocked.Exchange(ref _phase, PhaseResultReady); var context = _context; if (!IsClosed && context.StoppingToken.IsCancellationRequested) @@ -575,8 +545,6 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); } - await CompleteBatchAsync(result).ConfigureAwait(false); - _consumerObservedCompletion = true; return false; } catch (Exception ex) @@ -586,6 +554,84 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) } } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask ReadNextPublishedResultAsync() + { + CommandResult? result = _current; + while (_commandIndex < _commands.Count) + { + result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + _current = result; + _currentPublished = false; + if (!_commands.ItemRef(_commandIndex).SuppressEnumeration) + return result; + + var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); + var suppressedError = result.Error; + if (suppressedError is null && completeError is { } completionError) + suppressedError = completionError.Error; + if (suppressedError is not null) + { + var exception = PgErrorException.Create(suppressedError); + var cold = GetOrCreateColdState(); + Interlocked.CompareExchange(ref cold.TerminalException, exception, null); + cold.DrainError ??= exception; + Interlocked.Exchange(ref _phase, PhaseDraining); + NotifyDrainStarted(); + _consumerDetached = true; + await DrainAsync().ConfigureAwait(false); + throw exception; + } + + _commandIndex++; + } + + if (result is null) + throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); + await CompleteBatchAsync(result).ConfigureAwait(false); + _consumerObservedCompletion = true; + return null; + } + + CommandResult? ReadNextPublishedResult() + { + CommandResult? result = _current; + while (_commandIndex < _commands.Count) + { + result = ReadResult(_commandIndex); + _current = result; + _currentPublished = false; + if (!_commands.ItemRef(_commandIndex).SuppressEnumeration) + return result; + + var completeError = CompleteCurrentResult(); + var suppressedError = result.Error; + if (suppressedError is null && completeError is { } completionError) + suppressedError = completionError.Error; + if (suppressedError is not null) + { + var exception = PgErrorException.Create(suppressedError); + var cold = GetOrCreateColdState(); + Interlocked.CompareExchange(ref cold.TerminalException, exception, null); + cold.DrainError ??= exception; + Interlocked.Exchange(ref _phase, PhaseDraining); + NotifyDrainStarted(); + _consumerDetached = true; + Drain(); + throw exception; + } + + _commandIndex++; + } + + if (result is null) + throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); + CompleteBatch(result); + _consumerObservedCompletion = true; + return null; + } + // Reads through the command's execute prelude and initializes the protocol-static result. async ValueTask ReadResultAsync(int commandIndex) { @@ -596,11 +642,22 @@ async ValueTask ReadResultAsync(int commandIndex) throw context.FlowTerminationException; PgError? error; RowDescription? requestedRowDescription; - var describeOnly = _commands.ItemRef(commandIndex).DescribeOnly; - var hasPreparedDescription = _commands.ItemRef(commandIndex).Descriptor + ref readonly var command = ref _commands.ItemRef(commandIndex); + var describeOnly = command.DescribeOnly; + var hasPreparedDescription = command.Descriptor is { IsPrepared: true, PreparedRowDescription: not null }; - decoder.UseReadTimeout(_commands.ItemRef(commandIndex).Timeout); - if (hasPreparedDescription && !describeOnly) + decoder.UseReadTimeout(command.Timeout); + ParameterTypeList? preparationParameterTypes = null; + if (command.DescribeForPreparation) + { + var preparation = await command.ReadPreparationDescriptionAsync( + decoder, context.GetProtocolStatic().RowDescription) + .ConfigureAwait(false); + error = preparation.Item1; + preparationParameterTypes = preparation.Item2; + requestedRowDescription = preparation.Item3; + } + else if (hasPreparedDescription && !describeOnly) { // Prepared commands with a known description have the compact BindComplete -> // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes this @@ -629,11 +686,12 @@ async ValueTask ReadResultAsync(int commandIndex) } else { - (error, requestedRowDescription) = await _commands.ItemRef(commandIndex) + (error, requestedRowDescription) = await command .ReadUntilExecuteAsync(decoder, context.GetProtocolStatic().RowDescription) .ConfigureAwait(false); } - return InitializeResult(commandIndex, error, requestedRowDescription); + return InitializeResult( + commandIndex, error, requestedRowDescription, preparationParameterTypes); } CommandResult ReadResult(int commandIndex) @@ -644,13 +702,29 @@ CommandResult ReadResult(int commandIndex) throw context.FlowTerminationException; ref readonly var command = ref _commands.ItemRef(commandIndex); decoder.UseReadTimeout(command.Timeout); - var (error, requestedRowDescription) = command - .ReadUntilExecute(decoder, context.GetProtocolStatic().RowDescription); - return InitializeResult(commandIndex, error, requestedRowDescription); + PgError? error; + RowDescription? requestedRowDescription; + ParameterTypeList? preparationParameterTypes = null; + if (command.DescribeForPreparation) + { + var preparation = command.ReadPreparationDescription( + decoder, context.GetProtocolStatic().RowDescription); + error = preparation.Item1; + preparationParameterTypes = preparation.Item2; + requestedRowDescription = preparation.Item3; + } + else + { + (error, requestedRowDescription) = command.ReadUntilExecute( + decoder, context.GetProtocolStatic().RowDescription); + } + return InitializeResult( + commandIndex, error, requestedRowDescription, preparationParameterTypes); } CommandResult InitializeResult( - int commandIndex, PgError? error, RowDescription? requestedRowDescription) + int commandIndex, PgError? error, RowDescription? requestedRowDescription, + ParameterTypeList? preparationParameterTypes = null) { var context = _context; ref readonly var readState = ref context.GetProtocolStatic(); @@ -662,7 +736,8 @@ CommandResult InitializeResult( if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault && (error is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) { - descriptor = CommandDescriptor.CreatePrepared(descriptor.CommandName, descriptor.ParameterTypes, + descriptor = CommandDescriptor.CreatePrepared(descriptor.CommandName, + preparationParameterTypes ?? descriptor.ParameterTypes, requestedRowDescription?.Preserve()); } result.Initialize(this, commandIndex, descriptor, requestedRowDescription, @@ -735,10 +810,12 @@ void CompleteBatch(CommandResult result) // consumer must observe, then complete the pipeline task. void Finish(CommandResult result) { - if (result.Error is { } error && _consumerDetached && !IsOwnCancellation(error)) + if (result.Error is { } error && _consumerDetached && !_currentPublished + && !IsOwnCancellation(error)) GetOrCreateColdState().DrainError = PgErrorException.Create(error); _context.GetProtocolStatic().Reset(); _current = null; + _currentPublished = false; if (IsCancelRequested) Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, new OperationCanceledException(_coldState!.DeliverToken), null); @@ -759,6 +836,7 @@ void FaultFromOwner(Exception exception) return; DisposeRegistrations(); _current = null; + _currentPublished = false; if (HasDecoder) _context.GetProtocolStatic().Reset(); CompletePipelineTask(exception); @@ -1079,6 +1157,7 @@ protected override void OnReset() _commandIndex = -1; _context = default; _current = null; + _currentPublished = false; _readFlowRfq = false; _consumerDetached = false; _consumerObservedCompletion = false; From a93bee3443e34b878ca18c4e76e56a5ba8f6c3ce Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 00:58:03 +0200 Subject: [PATCH 086/136] Complete replacement flow cancellation and drain semantics --- Slon.Tests/Pg/CommandDrainTests.cs | 9 + Slon.Tests/Pg/CommandUserCancellationTests.cs | 6 + Slon.Tests/Pg/RacingDisposeInMemoryTests.cs | 30 ++ .../Pg/Protocol/Flows/CommandExecutionFlow.cs | 315 +++++++++++++++--- 4 files changed, 313 insertions(+), 47 deletions(-) diff --git a/Slon.Tests/Pg/CommandDrainTests.cs b/Slon.Tests/Pg/CommandDrainTests.cs index 08f2fc8..b899f8c 100644 --- a/Slon.Tests/Pg/CommandDrainTests.cs +++ b/Slon.Tests/Pg/CommandDrainTests.cs @@ -226,6 +226,9 @@ public async Task ConsumerDispose_MidBatch_BodyDrainsRemaining_ConnectionUsable( // wire usable (a hang shows as the "select 1" WaitAsync timing out, not a suite hang). [TestMethod] [DoNotParallelize] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises the legacy body coroutine's open-before-park rendezvous.")] +#endif public async Task ConsumerDispose_MidBatch_SyncDispose_OpenBeforePark_Stress() { var iters = StressEnv.Iterations(fallback: 8, cap: 8_000); @@ -250,6 +253,9 @@ public async Task ConsumerDispose_MidBatch_SyncDispose_OpenBeforePark_Stress() // CompleteEnumeration, or CompleteEnumerationWithException), completing the body cross-thread while the pump is parked - the // sticky terminal publication must wake it. No other live-server test reaches this interleaving. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises the legacy body coroutine's in-flight completion/pump handoff race.")] +#endif public async Task InFlightCompletion_RacesSyncDispose_PumpNeverStrands_Stress() { // Each iteration is a full connect + force-abort cycle. Cap it because this is path coverage, @@ -339,6 +345,9 @@ public async Task StoppingToken_MidBatch_FollowedByDispose_FlowCompletesCleanly( // gate await is faulted by the heartbeat-driven OnStopping and the consumer's MoveNext // surfaces PgClientClosedException without any delivery. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Requires the legacy body coroutine to read and publish a result before any consumer advances the flow.")] +#endif public async Task StoppingToken_PreFireAsync_BodyFaultsWithoutDelivery() { var protocol = await PgTestPool.NewIsolatedAsync(); diff --git a/Slon.Tests/Pg/CommandUserCancellationTests.cs b/Slon.Tests/Pg/CommandUserCancellationTests.cs index d7970dc..cf5c7ca 100644 --- a/Slon.Tests/Pg/CommandUserCancellationTests.cs +++ b/Slon.Tests/Pg/CommandUserCancellationTests.cs @@ -131,6 +131,9 @@ public async Task UserCt_FiresMidRead_SurfacesOce_ProtocolUsable() // Autonomous execution may enter the command read before the consumer supplies its per-read // token. The late token must still arm backend cancellation for that active read. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Requires the legacy body to enter a read before the consumer supplies its token; the replacement consumer owns the read from entry.")] +#endif public async Task UserCt_SuppliedAfterReadStarted_RequestsCancellation_ProtocolUsable() { await using var blocker = await PgAdvisoryLock.AcquireAsync(); @@ -571,6 +574,9 @@ public async Task ConsumerDispose_UsesItsOwnGraceBeforeSideChannelAttempt() } [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Requires a second body-owned drain read after the consumer read times out; the replacement retains one read owner through drain.")] +#endif public async Task ServerCancel_ReadTimeoutAfterAmbiguousRetryAbortsWire() { var iterations = StressEnv.Iterations(fallback: 1, cap: 5_000); diff --git a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs index c9af025..de43791 100644 --- a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs +++ b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs @@ -399,6 +399,9 @@ public Task RunLoop() => Task.Run(async () => // a forceful abort faults that read -> body's closed catch. Pre-fix: rethrow escaped DisposeAsync. // Now: DisposeAsyncCore swallows it. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises legacy body-driven throw and caller-gate ordering.")] +#endif public async Task Ordering1_BodyDrivenThrow_DisposeConverges() { await using var s = await BuildToFirstResultParked(); @@ -421,6 +424,9 @@ public async Task Ordering1_BodyDrivenThrow_DisposeConverges() // This isolates terminal publication and the continuation handoff without ThreadPool admission, // PostgreSQL timing, or an advisory lock. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises takeover of the legacy body coroutine during synchronous disposal.")] +#endif public async Task SyncDispose_InFlightReadFault_Converges() { var iterations = Math.Clamp(StressEnv.Iterations(fallback: 1, cap: int.MaxValue), 1, 500); @@ -449,6 +455,9 @@ public async Task SyncDispose_InFlightReadFault_Converges() // bounded exactly as production bounds it - advance past CompletionTimeout (30s) so the graceful->abort // escalation faults the parked drain read and DisposeAsync converges, swallowing the close. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises the legacy caller gate winning before the body throw.")] +#endif public async Task Ordering2_GateFirstThrow_DisposeConverges() { await using var s = await BuildToFirstResultParked(); @@ -473,6 +482,9 @@ public async Task Ordering2_GateFirstThrow_DisposeConverges() // consume that progress without receiving a continuation and transfer to AwaitDrainOnDispose. // Escalation must still redrive/fault the held read and complete both teardown participants. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises the legacy caller gate's progress-before-takeover ordering.")] +#endif public async Task SyncDispose_GateProgressBeforeTakeover_Converges() { var iterations = Math.Clamp(StressEnv.Iterations(fallback: 1, cap: int.MaxValue), 1, 500); @@ -500,6 +512,9 @@ public async Task SyncDispose_GateProgressBeforeTakeover_Converges() // progress-only wake; only afterwards does releasing the response let the body publish its sync // handoff continuation. Null progress cannot be mistaken for body termination. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises late publication of a legacy body handoff continuation.")] +#endif public async Task SyncDispose_ProgressWakeBeforeLateHandoff_DrivesBodyToTermination() { var clock = new FakeTimeProvider(); @@ -539,6 +554,9 @@ public async Task SyncDispose_ProgressWakeBeforeLateHandoff_DrivesBodyToTerminat // A sync-at-bind body parked between command results still owns a continuation after close becomes // consumer-terminal. Dispose must drive that continuation before returning or surfacing the close. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises the legacy body's inter-result caller gate.")] +#endif public async Task SyncFlow_CloseAtInterResultPark_DisposeRetainsDriveObligation() { var clock = new FakeTimeProvider(); @@ -583,6 +601,9 @@ public async Task SyncFlow_CloseAtInterResultPark_DisposeRetainsDriveObligation( // published CancelException, so the consumer's next MoveNextAsync self-delivers the close and the // loop converges. Asserts convergence (a hang here would regress that protection). [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises self-delivery after a no-op fault on the legacy caller gate.")] +#endif public async Task Ordering3_GateFaultNoOp_SelfDeliverConverges() { await using var s = await BuildToFirstResultParked(); @@ -607,6 +628,9 @@ public async Task Ordering3_GateFaultNoOp_SelfDeliverConverges() // abort lands CompleteEnumerationWithException on the LIVE (Reset) generation, because every body read in this flow // is preceded by a consumer Reset. Documents why the read-fault alone cannot produce the hang. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises the legacy body's read-fault-to-caller-gate transition.")] +#endif public async Task Ordering3_ReadFaultPath_NeverNoOps_Converges() { await using var s = await BuildToFirstResultParked(); @@ -632,6 +656,9 @@ public async Task Ordering3_ReadFaultPath_NeverNoOps_Converges() // inline ping-pong), so this is a convergence regression test, not an isolation of any one waker. The // never-started lost-completion (the actual reported hang) is gated by the stress repro + dump. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises graceful close while the legacy body is parked at its inter-result gate.")] +#endif public async Task MultiCommand_GracefulCloseAtInterResultGate_Converges() { await using var s = await BuildMultiToFirstResultParked(); @@ -664,6 +691,9 @@ public async Task MultiCommand_GracefulCloseAtInterResultGate_Converges() // the stale result 1. Ordering3_GateFaultNoOp asserts the read loop converges; a stale re-yield does // not hang, so this pins the specific next-call outcome the loop cannot catch. [TestMethod] +#if COMMAND_FLOW_NEXT + [Ignore("Exercises stale continuation suppression in the legacy caller gate.")] +#endif public async Task GateFaultBeforeNextMoveNext_SelfDeliversClose_NeverReYieldsStale() { await using var s = await BuildToFirstResultParked(); diff --git a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs index 8d476da..fe0cf0a 100644 --- a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs @@ -27,6 +27,7 @@ internal readonly struct CommandExecutionFlowOptions // CommandFlow contract; the eventual single-command specialization will remain a separate sealed type. internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource, IValueTaskSource { + static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); // Decoder ownership. Reading and Draining name a frame that owns the decoder. Initial and // ResultReady are idle states which exactly one party leaves by compare-exchange. const int PhaseInitial = 0; @@ -36,6 +37,12 @@ internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource _pipelineTaskSource; - // The submission token occupies this slot until a consumer attaches, after which the consumer - // token replaces it, including with default. Cancellation delivery, close, and failures stay cold. + // Flow-lifetime and per-read cancellation are distinct: the former reaches the remaining physical + // command list, while the latter is bounded to the read window in which the caller supplied it. CancellationToken _flowToken; CancellationTokenRegistration _flowRegistration; ColdState? _coldState; @@ -68,12 +75,20 @@ internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource? DrainErrors; } CommandExecutionFlow(bool async, TimeSpan? pendingTimeout = null) @@ -133,6 +148,10 @@ internal CommandExecutionFlow Initialize(bool async, in CommandExecutionFlowOpti private protected override FlowHandoffEvent? HandoffEvent => _handoffEvent; protected override bool EnableActivationTimeout => _enableActivationTimeout; protected override TimeSpan? PendingTimeout => _pendingTimeout; + internal override TimeSpan? BackendCancellationGracePeriod + => Volatile.Read(ref _consumerDetached) + ? ConsumerDrainCancellationGracePeriod + : null; internal override void BindCallerToken(CancellationToken cancellationToken) => _flowToken = cancellationToken; @@ -144,7 +163,9 @@ public Enumerator GetEnumerator() public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { - _flowToken = cancellationToken; + // A missing enumeration token must not erase the token captured when the flow was queued. + if (cancellationToken.CanBeCanceled) + _flowToken = cancellationToken; return new(this, cancellationToken); } @@ -350,6 +371,11 @@ bool First() var result = ReadNextPublishedResult(); return result is not null && PublishSynchronousResult(result); } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } catch (Exception ex) { FaultFromOwner(ex); @@ -365,6 +391,14 @@ bool NextBatch() var result = _current!; var completeError = CompleteCurrentResult(); _currentPublished = false; + if (Volatile.Read(ref _coldState)?.TerminalException is { } consumerFault) + { + Interlocked.Exchange(ref _phase, PhaseDraining); + NotifyDrainStarted(); + _consumerDetached = true; + Drain(); + ExceptionDispatchInfo.Throw(consumerFault); + } if (completeError is { TransactionStatus: TransactionStatus.Unknown }) SkipDiscardedCommands(); @@ -375,6 +409,11 @@ bool NextBatch() return false; } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } catch (Exception ex) { FaultFromOwner(ex); @@ -450,7 +489,7 @@ ValueTask MoveNextAsync(CancellationToken cancellationToken) // A pre-cancelled token releases the caller immediately. The wire still drains to RFQ. ValueTask CancelBeforeRead(CancellationToken cancellationToken) { - RequestCancel(cancellationToken); + RequestCancel(cancellationToken, CancellationScope.CurrentWindow); return ValueTask.FromException(new OperationCanceledException(cancellationToken)); } @@ -498,6 +537,11 @@ async ValueTask FirstAsync(CancellationToken cancellationToken) } deliver = Volatile.Read(ref _coldState)?.TerminalException; } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } catch (Exception ex) { FaultFromOwner(ex); @@ -519,6 +563,14 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) await resultEnumerator.DisposeAsync().ConfigureAwait(false); var completeError = resultEnumerator.CompleteError; _currentPublished = false; + if (Volatile.Read(ref _coldState)?.TerminalException is { } consumerFault) + { + Interlocked.Exchange(ref _phase, PhaseDraining); + NotifyDrainStarted(); + _consumerDetached = true; + await DrainAsync().ConfigureAwait(false); + ExceptionDispatchInfo.Throw(consumerFault); + } if (completeError is { TransactionStatus: TransactionStatus.Unknown }) await SkipDiscardedCommandsAsync().ConfigureAwait(false); @@ -547,6 +599,11 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) return false; } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } catch (Exception ex) { FaultFromOwner(ex); @@ -576,7 +633,11 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) var exception = PgErrorException.Create(suppressedError); var cold = GetOrCreateColdState(); Interlocked.CompareExchange(ref cold.TerminalException, exception, null); - cold.DrainError ??= exception; + (cold.DrainErrors ??= new()).Add(exception); + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + await SkipDiscardedCommandsAsync().ConfigureAwait(false); + _commandIndex++; + _current = null; Interlocked.Exchange(ref _phase, PhaseDraining); NotifyDrainStarted(); _consumerDetached = true; @@ -589,7 +650,7 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) if (result is null) throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); - await CompleteBatchAsync(result).ConfigureAwait(false); + await CompleteBatchAsync().ConfigureAwait(false); _consumerObservedCompletion = true; return null; } @@ -614,7 +675,11 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) var exception = PgErrorException.Create(suppressedError); var cold = GetOrCreateColdState(); Interlocked.CompareExchange(ref cold.TerminalException, exception, null); - cold.DrainError ??= exception; + (cold.DrainErrors ??= new()).Add(exception); + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + SkipDiscardedCommands(); + _commandIndex++; + _current = null; Interlocked.Exchange(ref _phase, PhaseDraining); NotifyDrainStarted(); _consumerDetached = true; @@ -627,7 +692,7 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) if (result is null) throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); - CompleteBatch(result); + CompleteBatch(); _consumerObservedCompletion = true; return null; } @@ -790,29 +855,26 @@ void ReadRfq() PgErrorException.Throw(rfqError); } - async ValueTask CompleteBatchAsync(CommandResult result) + async ValueTask CompleteBatchAsync() { if (_readFlowRfq) await ReadRfqAsync().ConfigureAwait(false); await DisposeRegistrationsAsync().ConfigureAwait(false); - Finish(result); + Finish(); } - void CompleteBatch(CommandResult result) + void CompleteBatch() { if (_readFlowRfq) ReadRfq(); DisposeRegistrations(); - Finish(result); + Finish(); } // The wire is at this command's RFQ. Release the shared read objects, record the outcome the // consumer must observe, then complete the pipeline task. - void Finish(CommandResult result) + void Finish() { - if (result.Error is { } error && _consumerDetached && !_currentPublished - && !IsOwnCancellation(error)) - GetOrCreateColdState().DrainError = PgErrorException.Create(error); _context.GetProtocolStatic().Reset(); _current = null; _currentPublished = false; @@ -877,20 +939,36 @@ async ValueTask DrainAsync() if (result is null) { await new ValueTask(this, _readySource.Version).ConfigureAwait(false); - _commandIndex = 0; + if (_commandIndex < 0) + _commandIndex = 0; + if (_commandIndex >= _commands.Count) + { + await CompleteBatchAsync().ConfigureAwait(false); + return; + } result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); } while (true) { var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); + CaptureDrainError(result, completeError); + _currentPublished = false; if (completeError is { TransactionStatus: TransactionStatus.Unknown }) await SkipDiscardedCommandsAsync().ConfigureAwait(false); if (++_commandIndex >= _commands.Count) break; result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); } - await CompleteBatchAsync(result).ConfigureAwait(false); + await CompleteBatchAsync().ConfigureAwait(false); + } + catch (TimeoutException ex) + { + // A timeout during semantic drain escalates the same cancellation episode immediately. + // The pipeline failure then hands any remaining wire obligation to recovery. + RequestCancel(default, CancellationScope.RemainingFlow, + BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); + FaultFromOwner(ex); } catch (Exception ex) { @@ -898,6 +976,22 @@ async ValueTask DrainAsync() } } + // A read timeout is terminal for the consumer but not for the wire obligation. This frame has + // released the decoder read tenure, so transfer ownership to an autonomous semantic drain while + // the original MoveNext returns its timeout immediately. + void HandleReadTimeout(TimeoutException exception) + { + Interlocked.CompareExchange( + ref GetOrCreateColdState().TerminalException, exception, null); + _consumerDetached = true; + NotifyDrainStarted(); + Interlocked.Exchange(ref _phase, PhaseDraining); + RequestCancel(default, CancellationScope.RemainingFlow, + BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); + ThreadPool.UnsafeQueueUserWorkItem( + static state => _ = ((CommandExecutionFlow)state!).DrainAsync(), this); + } + void Drain() { try @@ -908,20 +1002,28 @@ void Drain() // Synchronous disposal before any read. The activation bridge normally completed long // ago, so this bridge is rarely more than a status check. new ValueTask(this, _readySource.Version).AsTask().GetAwaiter().GetResult(); - _commandIndex = 0; + if (_commandIndex < 0) + _commandIndex = 0; + if (_commandIndex >= _commands.Count) + { + CompleteBatch(); + return; + } result = ReadResult(_commandIndex); } while (true) { var completeError = CompleteCurrentResult(); + CaptureDrainError(result, completeError); + _currentPublished = false; if (completeError is { TransactionStatus: TransactionStatus.Unknown }) SkipDiscardedCommands(); if (++_commandIndex >= _commands.Count) break; result = ReadResult(_commandIndex); } - CompleteBatch(result); + CompleteBatch(); } catch (Exception ex) { @@ -944,11 +1046,13 @@ ValueTask DisposeAsync() _consumerDetached = true; if (_current is { IsComplete: false } || _commandIndex + 1 < _commands.Count) - RequestCancel(default); + RequestCancel(default, CancellationScope.RemainingFlow); return WaitForDrainOnDispose ? DisposeDrainAsync() : FireAndForgetDrain(); case PhaseReading: - return ValueTask.FromException( - ThrowHelper.ThrowInvalidOperation("Cannot dispose the flow while a read is in progress.")); + _consumerDetached = true; + NotifyDrainStarted(); + RequestCancel(default, CancellationScope.RemainingFlow); + return WaitForDrainOnDispose ? DisposeCompletedAsync() : default; default: return !WaitForDrainOnDispose || _consumerObservedCompletion ? default @@ -973,8 +1077,7 @@ ValueTask FireAndForgetDrain() async ValueTask DisposeCompletedAsync() { await WaitForCompletionAsync().ConfigureAwait(false); - if (Volatile.Read(ref _coldState)?.DrainError is { } drainError) - throw drainError; + ThrowDrainErrors(); } // Flow completion is independent of errors accumulated while draining. A close is a clean @@ -1007,13 +1110,17 @@ void Dispose() _consumerDetached = true; if (_current is { IsComplete: false } || _commandIndex + 1 < _commands.Count) - RequestCancel(default); + RequestCancel(default, CancellationScope.RemainingFlow); Drain(); if (WaitForDrainOnDispose) DisposeCompleted(); return; case PhaseReading: - ThrowHelper.ThrowInvalidOperation("Cannot dispose the flow while a read is in progress."); + _consumerDetached = true; + NotifyDrainStarted(); + RequestCancel(default, CancellationScope.RemainingFlow); + if (WaitForDrainOnDispose) + DisposeCompleted(); return; default: if (WaitForDrainOnDispose && !_consumerObservedCompletion) @@ -1032,8 +1139,28 @@ void DisposeCompleted() catch (PgClientClosedException) { } - if (Volatile.Read(ref _coldState)?.DrainError is { } drainError) - throw drainError; + ThrowDrainErrors(); + } + + void CaptureDrainError(CommandResult result, + (PgError Error, TransactionStatus TransactionStatus)? completeError) + { + if (!_consumerDetached || _currentPublished) + return; + var error = result.Error ?? completeError?.Error; + if (error is null || IsOwnCancellation(error)) + return; + var cold = GetOrCreateColdState(); + (cold.DrainErrors ??= new()).Add(PgErrorException.Create(error)); + } + + void ThrowDrainErrors() + { + if (Volatile.Read(ref _coldState)?.DrainErrors is not { Count: > 0 } errors) + return; + if (errors.Count is 1) + ExceptionDispatchInfo.Throw(errors[0]); + throw new AggregateException(errors); } // When true, disposal waits for the drain to reach RFQ and for framework release. Otherwise it @@ -1042,56 +1169,147 @@ void DisposeCompleted() void RegisterCancellation(CancellationToken callerToken) { - if (callerToken == _flowToken && - (_flowRegistration != default || !callerToken.CanBeCanceled)) - return; - var registration = _flowRegistration; - _flowRegistration = default; - registration.Dispose(); - _flowToken = callerToken; - if (callerToken.CanBeCanceled) - _flowRegistration = callerToken.UnsafeRegister(static (state, token) - => ((CommandExecutionFlow)state!).RequestCancel(token), this); + // Keep the second token/registration pair off ordinary flow objects. Default-token traffic + // does not need cancellation state at all and remains the allocation/footprint hot path. + var cancellation = Volatile.Read(ref _coldState); + if (callerToken.CanBeCanceled || cancellation is not null) + { + cancellation ??= GetOrCreateColdState(); + if (callerToken != cancellation.CallerToken) + { + var callerRegistration = cancellation.CallerRegistration; + cancellation.CallerRegistration = default; + callerRegistration.Dispose(); + cancellation.CallerToken = callerToken; + } + if (callerToken.CanBeCanceled && cancellation.CallerRegistration == default) + cancellation.CallerRegistration = callerToken.UnsafeRegister(static (state, token) + => ((CommandExecutionFlow)state!).RequestCancel( + token, CancellationScope.CurrentWindow), this); + } + + if (_flowToken.CanBeCanceled && _flowRegistration == default) + _flowRegistration = _flowToken.UnsafeRegister(static (state, token) + => ((CommandExecutionFlow)state!).RequestCancel( + token, CancellationScope.RemainingFlow), this); } ValueTask DisposeRegistrationsAsync() { - if (_flowRegistration == default) + var cancellation = Volatile.Read(ref _coldState); + var callerRegistration = cancellation?.CallerRegistration ?? default; + if (callerRegistration == default && _flowRegistration == default) return default; + if (cancellation is not null) + cancellation.CallerRegistration = default; var flowRegistration = _flowRegistration; _flowRegistration = default; - return flowRegistration.DisposeAsync(); + return DisposeRegistrationsAsync(callerRegistration, flowRegistration); + + static async ValueTask DisposeRegistrationsAsync( + CancellationTokenRegistration callerRegistration, + CancellationTokenRegistration flowRegistration) + { + await callerRegistration.DisposeAsync().ConfigureAwait(false); + await flowRegistration.DisposeAsync().ConfigureAwait(false); + } } void DisposeRegistrations() { + var cancellation = Volatile.Read(ref _coldState); + var callerRegistration = cancellation?.CallerRegistration ?? default; + if (cancellation is not null) + cancellation.CallerRegistration = default; var flowRegistration = _flowRegistration; _flowRegistration = default; + callerRegistration.Dispose(); flowRegistration.Dispose(); } // Cancellation only latches intent and requests a backend cancel. The frame owning the decoder // delivers it after the wire is back at RFQ. An idle flow drains autonomously first. - void RequestCancel(CancellationToken token) + void RequestCancel(CancellationToken token, CancellationScope scope, + BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, + BackendCancellationTiming subsequentTiming = BackendCancellationTiming.AfterGrace) { if (Volatile.Read(ref _phase) == PhaseCompleted) return; var cancellation = GetOrCreateColdState(); cancellation.DeliverToken = token; + RaiseCancellationScope(cancellation, scope); + RaiseCancellationTiming(ref cancellation.Timing, timing); + RaiseCancellationTiming(ref cancellation.SubsequentTiming, subsequentTiming); Interlocked.Exchange(ref cancellation.CancelRequested, true); if (HasDecoder) RequestBackendCancellation(); TryTakeOverDrain(); } + static void RaiseCancellationScope(ColdState cancellation, CancellationScope scope) + { + var requested = (int)scope; + var current = Volatile.Read(ref cancellation.Scope); + while (current < requested) + { + var observed = Interlocked.CompareExchange( + ref cancellation.Scope, requested, current); + if (observed == current) + return; + current = observed; + } + } + + static void RaiseCancellationTiming( + ref int location, BackendCancellationTiming timing) + { + var requested = (int)timing; + var current = Volatile.Read(ref location); + while (current < requested) + { + var observed = Interlocked.CompareExchange(ref location, requested, current); + if (observed == current) + return; + current = observed; + } + } + internal Task CancelAsync() { - RequestCancel(default); - return WaitForComplete().AsTask(); + var cancellation = GetOrCreateColdState(); + var delivery = Volatile.Read(ref cancellation.Delivery) + ?? Interlocked.CompareExchange(ref cancellation.Delivery, + new(TaskCreationOptions.RunContinuationsAsynchronously), null) + ?? cancellation.Delivery; + if (Volatile.Read(ref _phase) is PhaseCompleted) + { + delivery.TrySetResult(); + return delivery.Task; + } + RequestCancel(default, CancellationScope.RemainingFlow); + if (Volatile.Read(ref _phase) is PhaseCompleted) + delivery.TrySetResult(); + return delivery.Task; } void RequestBackendCancellation() - => _context.RequestBackendCancellation(this, CancellationWindow, BackendCancellationTiming.AfterGrace); + { + var cancellation = GetOrCreateColdState(); + var episodeKey = Volatile.Read(ref cancellation.EpisodeKey); + if (episodeKey is null) + { + var created = new object(); + episodeKey = Interlocked.CompareExchange( + ref cancellation.EpisodeKey, created, null) ?? created; + } + _context.RequestBackendCancellation( + this, CancellationWindow, + (BackendCancellationTiming)Volatile.Read(ref cancellation.Timing), + Volatile.Read(ref cancellation.Delivery), + episodeKey, + Math.Max(Volatile.Read(ref cancellation.Scope), (int)CancellationScope.CurrentWindow), + (BackendCancellationTiming)Volatile.Read(ref cancellation.SubsequentTiming)); + } // Finish and FaultFromOwner reset the shared read objects before the pipeline task completes, and // Current is null once the consumer observed the terminal, so nothing outlives the flow. @@ -1141,6 +1359,7 @@ internal override void Fail(Exception exception) protected override void OnReleasing(Exception? exception) { + Volatile.Read(ref _coldState)?.Delivery?.TrySetResult(); DisposeRegistrations(); _commands.Return(); } @@ -1165,6 +1384,8 @@ protected override void OnReset() _pipelineTaskSource.Reset(); _readyCompletion = 0; _drainStarted = 0; + _flowToken = default; + _flowRegistration = default; _coldState = null; _syncHandoffClaimed = false; _handoffEvent?.ResetInteraction(); From 592b315a097f6c74c87a49bbfe2c9916872287b8 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 01:36:00 +0200 Subject: [PATCH 087/136] Extract the reusable command execution core --- .../Pg/Protocol/Flows/CommandExecutionFlow.cs | 781 ++++++++++-------- Slon/Pg/Protocol/PgClientFlow.cs | 6 +- 2 files changed, 435 insertions(+), 352 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs index fe0cf0a..1c05887 100644 --- a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; using System.Threading.Tasks.Sources; using Slon.Runtime.CompilerServices; @@ -22,82 +23,80 @@ internal readonly struct CommandExecutionFlowOptions public TimeSpan? PendingTimeout { get; init; } } +internal sealed class CommandExecutionColdState +{ + internal bool CancelRequested; + internal CancellationToken CallerToken; + internal CancellationTokenRegistration CallerRegistration; + internal CancellationToken DeliverToken; + internal int Scope; + internal int Timing; + internal int SubsequentTiming; + internal TaskCompletionSource? Delivery; + internal object? EpisodeKey; + internal Exception? CloseException; + // Replayed by later consumer calls once the flow reached its terminal. + internal Exception? TerminalException; + // Command errors observed while draining without a consumer. Multiple Sync windows may each + // produce an independent ErrorResponse, all of which belong to the waiting disposer. + internal List? DrainErrors; +} + +internal enum CommandExecutionCancellationScope : byte +{ + CurrentWindow = 1, + RemainingFlow = 2 +} + +// Mutable execution state is stored inline by each concrete host. Core algorithms re-enter this +// field through their host-specific ops value after every await; they never mutate a copied struct. +[StructLayout(LayoutKind.Auto)] +internal struct CommandExecutionState +{ + internal int Phase; + internal CommandList Commands; + internal TimeSpan? PendingTimeout; + internal int CommandIndex; + internal PgClientFlow.Context Context; + internal bool ContextPublished; + internal CommandResult? Current; + internal bool CurrentPublished; + internal bool ReadFlowRfq; + internal bool ConsumerDetached; + internal bool ConsumerObservedCompletion; + internal Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore ReadySource; + internal int ReadyCompletion; + internal Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore PipelineTaskSource; + internal CancellationToken FlowToken; + internal CancellationTokenRegistration FlowRegistration; + internal CommandExecutionColdState? ColdState; + internal FlowHandoffEvent? HandoffEvent; + internal bool SyncHandoffClaimed; + internal int DrainStarted; + internal bool EnableActivationTimeout; + internal bool WaitForDrainOnDispose; +} + // Replacement-flow prototype: one consumer-owned decoder lifecycle for synchronous and asynchronous // execution, with the general multi-command result shape. Kept internal until it covers the complete // CommandFlow contract; the eventual single-command specialization will remain a separate sealed type. -internal sealed class CommandExecutionFlow : PgClientFlow, IValueTaskSource, IValueTaskSource +internal sealed partial class CommandExecutionFlow : PgClientFlow, IValueTaskSource, IValueTaskSource { static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); - // Decoder ownership. Reading and Draining name a frame that owns the decoder. Initial and - // ResultReady are idle states which exactly one party leaves by compare-exchange. - const int PhaseInitial = 0; - const int PhaseReading = 1; - const int PhaseResultReady = 2; - const int PhaseDraining = 3; - const int PhaseCompleted = 4; - int _phase; - - enum CancellationScope : byte - { - CurrentWindow = 1, - RemainingFlow = 2 - } - - CommandList _commands; - TimeSpan? _pendingTimeout; + CommandExecutionState _state; CommandExecutionFlowObserver? _commandObserver; object? _commandObserverState; - int _commandIndex = -1; - Context _context; - CommandResult? _current; - bool _currentPublished; - bool _readFlowRfq; - // Set by the consumer once it has started reading, so a drain knows whether to publish nothing. - bool _consumerDetached; - bool _consumerObservedCompletion; - - // Completed once the request is written and activation settled, faulted by teardown before then. - Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _readySource; - int _readyCompletion; - // The framework's pipeline task, completed by whichever frame consumes RFQ. - Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _pipelineTaskSource; - - // Flow-lifetime and per-read cancellation are distinct: the former reaches the remaining physical - // command list, while the latter is bounded to the read window in which the caller supplied it. - CancellationToken _flowToken; - CancellationTokenRegistration _flowRegistration; - ColdState? _coldState; - FlowHandoffEvent? _handoffEvent; - bool _syncHandoffClaimed; - int _drainStarted; - bool _enableActivationTimeout = true; - - sealed class ColdState - { - internal bool CancelRequested; - internal CancellationToken CallerToken; - internal CancellationTokenRegistration CallerRegistration; - internal CancellationToken DeliverToken; - internal int Scope; - internal int Timing; - internal int SubsequentTiming; - internal TaskCompletionSource? Delivery; - internal object? EpisodeKey; - internal Exception? CloseException; - // Replayed by later consumer calls once the flow reached its terminal. - internal Exception? TerminalException; - // Command errors observed while draining without a consumer. Multiple Sync windows may each - // produce an independent ErrorResponse, all of which belong to the waiting disposer. - internal List? DrainErrors; - } CommandExecutionFlow(bool async, TimeSpan? pendingTimeout = null) : base(supportsDeferredFlush: true) { - _pendingTimeout = pendingTimeout; + _state.CommandIndex = -1; + _state.EnableActivationTimeout = true; + _state.WaitForDrainOnDispose = true; + _state.PendingTimeout = pendingTimeout; IsAsync = async; if (!async) - _handoffEvent = new(false); + _state.HandoffEvent = new(false); } internal CommandExecutionFlow(bool async, params ReadOnlySpan commands) @@ -107,7 +106,7 @@ internal CommandExecutionFlow(bool async, params ReadOnlySpan commands) internal CommandExecutionFlow( bool async, bool enableActivationTimeout, params ReadOnlySpan commands) : this(async, commands) - => _enableActivationTimeout = enableActivationTimeout; + => _state.EnableActivationTimeout = enableActivationTimeout; internal CommandExecutionFlow(bool async, CommandList commands, TimeSpan? pendingTimeout = null) : this(async, pendingTimeout) @@ -128,12 +127,12 @@ internal CommandExecutionFlow Initialize(bool async, in CommandExecutionFlowOpti { IsAsync = async; if (!async) - _handoffEvent ??= new(false); + _state.HandoffEvent ??= new(false); var commands = options.Commands; if (commands.Count is 0) return this; - _commands = commands; - _pendingTimeout = options.PendingTimeout; + _state.Commands = commands; + _state.PendingTimeout = options.PendingTimeout; _commandObserver = options.Observer; _commandObserverState = options.ObserverState; if (options.Observer is { } observer) @@ -145,18 +144,18 @@ internal CommandExecutionFlow Initialize(bool async, in CommandExecutionFlowOpti } internal override bool DefersSyncHandoff => true; - private protected override FlowHandoffEvent? HandoffEvent => _handoffEvent; - protected override bool EnableActivationTimeout => _enableActivationTimeout; - protected override TimeSpan? PendingTimeout => _pendingTimeout; + private protected override FlowHandoffEvent? HandoffEvent => _state.HandoffEvent; + protected override bool EnableActivationTimeout => _state.EnableActivationTimeout; + protected override TimeSpan? PendingTimeout => _state.PendingTimeout; internal override TimeSpan? BackendCancellationGracePeriod - => Volatile.Read(ref _consumerDetached) + => Volatile.Read(ref _state.ConsumerDetached) ? ConsumerDrainCancellationGracePeriod : null; internal override void BindCallerToken(CancellationToken cancellationToken) - => _flowToken = cancellationToken; + => _state.FlowToken = cancellationToken; internal override CancellationToken MigrationCancellationToken - => _flowToken; + => _state.FlowToken; public Enumerator GetEnumerator() => new(this, default); @@ -165,17 +164,95 @@ public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = defau { // A missing enumeration token must not erase the token captured when the flow was queued. if (cancellationToken.CanBeCanceled) - _flowToken = cancellationToken; + _state.FlowToken = cancellationToken; return new(this, cancellationToken); } - internal CommandResult? CurrentResult => _current; - internal bool IsResultReady => Volatile.Read(ref _phase) is PhaseResultReady; - internal int VisibleCommandCount => _commands.Count; + internal CommandResult? CurrentResult => _state.Current; + internal bool IsResultReady => Core.IsResultReady; + internal int VisibleCommandCount => _state.Commands.Count; internal ValueTask MoveNextResultAsync(CancellationToken cancellationToken) - => MoveNextAsync(cancellationToken); - internal void DisposeResults() => Dispose(); - internal ValueTask DisposeResultsAsync() => DisposeAsync(); + => Core.MoveNextAsync(cancellationToken); + internal void DisposeResults() => Core.Dispose(); + internal ValueTask DisposeResultsAsync() => Core.DisposeAsync(); + internal bool WaitForDrainOnDispose + { + get => _state.WaitForDrainOnDispose; + set => _state.WaitForDrainOnDispose = value; + } + + CommandExecutionCore Core => new(new(this)); + + internal ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) + => Core.ConsumeNonQueryAsync(cancellationToken); + + protected override ValueTask ExecuteAuto(Context context) + => Core.ExecuteAuto(context); + + internal Task CancelAsync() => Core.CancelAsync(); + + internal override bool ResetsSharedReadStateBeforeRelease => true; + protected override void OnStopping(Exception exception) => Core.OnStopping(exception); + protected override void OnAbort(Exception exception) => Core.OnAbort(exception); + internal override void Fail(Exception exception) => Core.Fail(exception); + protected override void OnReleasing(Exception? exception) => Core.OnReleasing(exception); + protected override void OnDiscarded() => Core.OnDiscarded(); + protected override void OnReset() => Core.OnReset(); + + readonly struct Ops(CommandExecutionFlow owner) : ICommandExecutionFlowOps + { + readonly CommandExecutionFlow _owner = owner; + + public static Ops Create(PgClientFlow flow) => new((CommandExecutionFlow)flow); + public PgClientFlow Flow => _owner; + public ref CommandExecutionState State => ref _owner._state; + public bool IsAsync + { + get => _owner.IsAsync; + set => _owner.IsAsync = value; + } + public bool IsAsyncAtDispatch => _owner.IsAsyncAtDispatch; + public bool HasSuccessfulActivation => _owner.HasSuccessfulActivation; + public void WaitForSyncHandoff() => _owner.WaitForSyncHandoff(); + public void OnCommandResult(CommandResult result) + => _owner._commandObserver?.OnCommandResult( + _owner, result, _owner._commandObserverState); + public void OnDrainStarted() + => _owner._commandObserver?.OnDrainStarted( + _owner, _owner._commandObserverState); + public void OnDiscarded() + => _owner.GetObserver(out var observerState)?.OnCompleting( + _owner, null, observerState); + } +} + +internal interface ICommandExecutionFlowOps + where TSelf : struct, ICommandExecutionFlowOps +{ + static abstract TSelf Create(PgClientFlow flow); + PgClientFlow Flow { get; } + ref CommandExecutionState State { get; } + bool IsAsync { get; set; } + bool IsAsyncAtDispatch { get; } + bool HasSuccessfulActivation { get; } + void WaitForSyncHandoff(); + void OnCommandResult(CommandResult result); + void OnDrainStarted(); + void OnDiscarded(); +} + +readonly struct CommandExecutionCore(TOps ops) + where TOps : struct, ICommandExecutionFlowOps +{ + const int PhaseInitial = 0; + const int PhaseReading = 1; + const int PhaseResultReady = 2; + const int PhaseDraining = 3; + const int PhaseCompleted = 4; + + readonly TOps _ops = ops; + ref CommandExecutionState _state => ref _ops.State; + internal bool IsResultReady => Volatile.Read(ref _state.Phase) is PhaseResultReady; [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] @@ -183,15 +260,16 @@ internal async ValueTask ConsumeNonQueryAsync( CancellationToken cancellationToken = default) { var recordsAffected = -1L; - var results = GetAsyncEnumerator(cancellationToken); + if (cancellationToken.CanBeCanceled) + _state.FlowToken = cancellationToken; try { - while (await results.MoveNextAsync().ConfigureAwait(false)) + while (await MoveNextAsync(cancellationToken).ConfigureAwait(false)) { - var result = results.Current; + var result = _state.Current!; await result.CompleteAsync().ConfigureAwait(false); - result.GetCommandComplete(); - if (result.RecordsAffected is { } affected and >= 0) + var affected = result.GetCommandComplete().BatchRecordsAffected; + if (affected >= 0) recordsAffected = recordsAffected < 0 ? affected : checked(recordsAffected + affected); @@ -200,31 +278,32 @@ internal async ValueTask ConsumeNonQueryAsync( } finally { - await results.DisposeAsync().ConfigureAwait(false); + await DisposeAsync().ConfigureAwait(false); } } - ColdState GetOrCreateColdState() - => Volatile.Read(ref _coldState) ?? - Interlocked.CompareExchange(ref _coldState, new(), null) ?? _coldState; + CommandExecutionColdState GetOrCreateColdState() + => Volatile.Read(ref _state.ColdState) ?? + Interlocked.CompareExchange(ref _state.ColdState, new(), null) ?? _state.ColdState; - bool IsClosed => Volatile.Read(ref _coldState)?.CloseException is not null; - bool IsCancelRequested => Volatile.Read(ref _coldState) is { CancelRequested: true }; - bool HasDecoder => HasSuccessfulActivation; + bool IsClosed => Volatile.Read(ref _state.ColdState)?.CloseException is not null; + bool IsCancelRequested => Volatile.Read(ref _state.ColdState) is { CancelRequested: true }; + bool HasDecoder => _state.ContextPublished && _ops.HasSuccessfulActivation; - protected override ValueTask ExecuteAuto(Context context) + internal ValueTask ExecuteAuto(PgClientFlow.Context context) { - _context = context; + _state.Context = context; + _state.ContextPublished = true; ValueTask writeTask; try { - ref readonly var template = ref _commands.ItemRef(_commands.Count - 1); + ref readonly var template = ref _state.Commands.ItemRef(_state.Commands.Count - 1); var appendSync = !template.WithSync; - _readFlowRfq = appendSync; + _state.ReadFlowRfq = appendSync; // Caller cancellation never cancels wire I/O. The consumer observes the latched intent and // drains its command to RFQ instead. - writeTask = IsAsync - ? _commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) + writeTask = _ops.IsAsync + ? _state.Commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) : WriteCommandsResumable(context, appendSync); // Observe synchronous faults here; pending writes remain the framework-owned trailing task. if (writeTask.IsCompleted) @@ -244,17 +323,19 @@ protected override ValueTask ExecuteAuto(Context context) if (activation.IsCompleted) OnActivationSettled(onExecutorStrand: true); else - activation.UnsafeOnCompleted(static state => ((CommandExecutionFlow)state!).OnActivationSettled(onExecutorStrand: false), this); - return new(new FlowTasks(writeTask, new ValueTask(this, _pipelineTaskSource.Version))); + activation.UnsafeOnCompleted(static state => + new CommandExecutionCore(TOps.Create((PgClientFlow)state!)) + .OnActivationSettled(onExecutorStrand: false), _ops.Flow); + return new(new FlowTasks(writeTask, new ValueTask((IValueTaskSource)_ops.Flow, _state.PipelineTaskSource.Version))); } [MethodImpl(MethodImplOptions.NoInlining)] - ValueTask WriteCommandsResumable(Context context, bool appendSync) + ValueTask WriteCommandsResumable(PgClientFlow.Context context, bool appendSync) { var encoder = context.GetEncoder(); ValueTask writeTask; using (encoder.BeginResumableWriteScope()) - writeTask = _commands.WriteCommandsResumable(encoder, appendSync); + writeTask = _state.Commands.WriteCommandsResumable(encoder, appendSync); return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); } @@ -265,7 +346,7 @@ void OnActivationSettled(bool onExecutorStrand) { try { - _ = _context.GetDecoderAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + _ = _state.Context.GetDecoderAsync().ConfigureAwait(false).GetAwaiter().GetResult(); } catch (Exception fault) { @@ -299,62 +380,62 @@ void FaultReady(Exception exception) bool CompleteReady(Exception? exception, bool runContinuationsAsynchronously) { - if (Interlocked.CompareExchange(ref _readyCompletion, 1, 0) != 0) + if (Interlocked.CompareExchange(ref _state.ReadyCompletion, 1, 0) != 0) return false; if (exception is null) - _readySource.SetResult(true, runContinuationsAsynchronously); + _state.ReadySource.SetResult(true, runContinuationsAsynchronously); else - _readySource.SetException(exception, runContinuationsAsynchronously); + _state.ReadySource.SetException(exception, runContinuationsAsynchronously); return true; } void CompletePipelineTask(Exception? exception, bool runContinuationsAsynchronously = false) { - if (Interlocked.Exchange(ref _phase, PhaseCompleted) is PhaseCompleted) + if (Interlocked.Exchange(ref _state.Phase, PhaseCompleted) is PhaseCompleted) return; if (exception is null) - _pipelineTaskSource.SetResult(true, runContinuationsAsynchronously); + _state.PipelineTaskSource.SetResult(true, runContinuationsAsynchronously); else - _pipelineTaskSource.SetException(exception, runContinuationsAsynchronously); + _state.PipelineTaskSource.SetException(exception, runContinuationsAsynchronously); } void EnsureSyncHandoff() { - if (IsAsyncAtDispatch) + if (_ops.IsAsyncAtDispatch) ThrowHelper.ThrowInvalidOperation( "Synchronous result consumption requires a flow initialized for synchronous execution."); - if (_syncHandoffClaimed) + if (_state.SyncHandoffClaimed) return; - WaitForSyncHandoff(); - _syncHandoffClaimed = true; + _ops.WaitForSyncHandoff(); + _state.SyncHandoffClaimed = true; } - bool MoveNext() + internal bool MoveNext() { EnsureSyncHandoff(); while (true) { - var phase = Volatile.Read(ref _phase); + var phase = Volatile.Read(ref _state.Phase); switch (phase) { case PhaseInitial: - if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseInitial) != PhaseInitial) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseInitial) != PhaseInitial) continue; - _commandIndex = 0; + _state.CommandIndex = 0; return First(); case PhaseResultReady: - if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) != PhaseResultReady) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) != PhaseResultReady) continue; return NextBatch(); case PhaseReading: ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow."); return false; case PhaseDraining: - WaitForCompleteSynchronously(); - throw Volatile.Read(ref _coldState)?.TerminalException + _ops.Flow.WaitForCompleteSynchronously(); + throw Volatile.Read(ref _state.ColdState)?.TerminalException ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); default: - if (Volatile.Read(ref _coldState)?.TerminalException is { } terminal) + if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } terminal) ExceptionDispatchInfo.Throw(terminal); return false; } @@ -366,7 +447,7 @@ bool First() try { WaitForReadySynchronously(); - Debug.Assert(!_consumerDetached); + Debug.Assert(!_state.ConsumerDetached); RegisterCancellation(default); var result = ReadNextPublishedResult(); return result is not null && PublishSynchronousResult(result); @@ -388,21 +469,21 @@ bool NextBatch() try { RegisterCancellation(default); - var result = _current!; + var result = _state.Current!; var completeError = CompleteCurrentResult(); - _currentPublished = false; - if (Volatile.Read(ref _coldState)?.TerminalException is { } consumerFault) + _state.CurrentPublished = false; + if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } consumerFault) { - Interlocked.Exchange(ref _phase, PhaseDraining); + Interlocked.Exchange(ref _state.Phase, PhaseDraining); NotifyDrainStarted(); - _consumerDetached = true; + _state.ConsumerDetached = true; Drain(); ExceptionDispatchInfo.Throw(consumerFault); } if (completeError is { TransactionStatus: TransactionStatus.Unknown }) SkipDiscardedCommands(); - _commandIndex++; + _state.CommandIndex++; var next = ReadNextPublishedResult(); if (next is not null) return PublishSynchronousResult(next); @@ -423,54 +504,54 @@ bool NextBatch() bool PublishSynchronousResult(CommandResult result) { - _current = result; - _currentPublished = true; - Interlocked.Exchange(ref _phase, PhaseResultReady); - var context = _context; + _state.Current = result; + _state.CurrentPublished = true; + Interlocked.Exchange(ref _state.Phase, PhaseResultReady); + var context = _state.Context; if (!IsClosed && context.StoppingToken.IsCancellationRequested) Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, context.FlowTerminationException, null); if (!IsCancelRequested && !IsClosed) return true; - if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) Drain(); else - WaitForCompleteSynchronously(); - throw Volatile.Read(ref _coldState)?.TerminalException + _ops.Flow.WaitForCompleteSynchronously(); + throw Volatile.Read(ref _state.ColdState)?.TerminalException ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); } void WaitForReadySynchronously() { - var ready = new ValueTask(this, _readySource.Version); + var ready = new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version); if (ready.IsCompleted) _ = ready.GetAwaiter().GetResult(); else _ = ready.AsTask().GetAwaiter().GetResult(); } - ValueTask MoveNextAsync(CancellationToken cancellationToken) + internal ValueTask MoveNextAsync(CancellationToken cancellationToken) { - if (!IsAsyncAtDispatch) + if (!_ops.IsAsyncAtDispatch) return ValueTask.FromException(ThrowHelper.ThrowInvalidOperation( "Asynchronous result consumption requires a flow initialized for asynchronous execution.")); while (true) { - var phase = Volatile.Read(ref _phase); + var phase = Volatile.Read(ref _state.Phase); switch (phase) { case PhaseInitial: if (cancellationToken.IsCancellationRequested) return CancelBeforeRead(cancellationToken); - if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseInitial) != PhaseInitial) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseInitial) != PhaseInitial) continue; - _commandIndex = 0; + _state.CommandIndex = 0; return FirstAsync(cancellationToken); case PhaseResultReady: if (cancellationToken.IsCancellationRequested) return CancelBeforeRead(cancellationToken); - if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) != PhaseResultReady) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) != PhaseResultReady) continue; return NextBatchAsync(cancellationToken); case PhaseReading: @@ -479,7 +560,7 @@ ValueTask MoveNextAsync(CancellationToken cancellationToken) case PhaseDraining: return AwaitTakeoverAsync(); default: - return Volatile.Read(ref _coldState)?.TerminalException is { } terminal + return Volatile.Read(ref _state.ColdState)?.TerminalException is { } terminal ? ValueTask.FromException(terminal) : new(false); } @@ -489,7 +570,7 @@ ValueTask MoveNextAsync(CancellationToken cancellationToken) // A pre-cancelled token releases the caller immediately. The wire still drains to RFQ. ValueTask CancelBeforeRead(CancellationToken cancellationToken) { - RequestCancel(cancellationToken, CancellationScope.CurrentWindow); + RequestCancel(cancellationToken, CommandExecutionCancellationScope.CurrentWindow); return ValueTask.FromException(new OperationCanceledException(cancellationToken)); } @@ -497,7 +578,7 @@ ValueTask CancelBeforeRead(CancellationToken cancellationToken) async ValueTask AwaitTakeoverAsync() { await WaitForCompletionAsync().ConfigureAwait(false); - throw Volatile.Read(ref _coldState)?.TerminalException ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); + throw Volatile.Read(ref _state.ColdState)?.TerminalException ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); } [RuntimeAsyncMethodGeneration(false)] @@ -507,26 +588,26 @@ async ValueTask FirstAsync(CancellationToken cancellationToken) Exception? deliver; try { - await new ValueTask(this, _readySource.Version).ConfigureAwait(false); - Debug.Assert(!_consumerDetached); + await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); + Debug.Assert(!_state.ConsumerDetached); RegisterCancellation(cancellationToken); var result = await ReadNextPublishedResultAsync().ConfigureAwait(false); if (result is null) return false; - _current = result; - _currentPublished = true; + _state.Current = result; + _state.CurrentPublished = true; // Publish the idle state, then recheck the latches. A latch that landed between the read // and this publication found no idle owner to take over, so this frame must act on it. - Interlocked.Exchange(ref _phase, PhaseResultReady); + Interlocked.Exchange(ref _state.Phase, PhaseResultReady); // Graceful stopping faults a result that arrives after the close began, as the ordinary // flow does at each result boundary. Latch it so the drain delivers that close. - var context = _context; + var context = _state.Context; if (!IsClosed && context.StoppingToken.IsCancellationRequested) Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, context.FlowTerminationException, null); if (!IsCancelRequested && !IsClosed) return true; - if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) { await DrainAsync().ConfigureAwait(false); } @@ -535,7 +616,7 @@ async ValueTask FirstAsync(CancellationToken cancellationToken) // The latching side took the decoder first. Park behind its drain. await WaitForCompletionAsync().ConfigureAwait(false); } - deliver = Volatile.Read(ref _coldState)?.TerminalException; + deliver = Volatile.Read(ref _state.ColdState)?.TerminalException; } catch (TimeoutException ex) { @@ -557,43 +638,43 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) try { RegisterCancellation(cancellationToken); - var result = _current!; - var resultEnumerator = _context.GetProtocolStatic() + var result = _state.Current!; + var resultEnumerator = _state.Context.GetProtocolStatic() .ResultMessageEnumerator; await resultEnumerator.DisposeAsync().ConfigureAwait(false); var completeError = resultEnumerator.CompleteError; - _currentPublished = false; - if (Volatile.Read(ref _coldState)?.TerminalException is { } consumerFault) + _state.CurrentPublished = false; + if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } consumerFault) { - Interlocked.Exchange(ref _phase, PhaseDraining); + Interlocked.Exchange(ref _state.Phase, PhaseDraining); NotifyDrainStarted(); - _consumerDetached = true; + _state.ConsumerDetached = true; await DrainAsync().ConfigureAwait(false); ExceptionDispatchInfo.Throw(consumerFault); } if (completeError is { TransactionStatus: TransactionStatus.Unknown }) await SkipDiscardedCommandsAsync().ConfigureAwait(false); - _commandIndex++; + _state.CommandIndex++; var next = await ReadNextPublishedResultAsync().ConfigureAwait(false); if (next is not null) { result = next; - _current = result; - _currentPublished = true; - Interlocked.Exchange(ref _phase, PhaseResultReady); - var context = _context; + _state.Current = result; + _state.CurrentPublished = true; + Interlocked.Exchange(ref _state.Phase, PhaseResultReady); + var context = _state.Context; if (!IsClosed && context.StoppingToken.IsCancellationRequested) Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, context.FlowTerminationException, null); if (!IsCancelRequested && !IsClosed) return true; - if (Interlocked.CompareExchange(ref _phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) await DrainAsync().ConfigureAwait(false); else await WaitForCompletionAsync().ConfigureAwait(false); - throw Volatile.Read(ref _coldState)?.TerminalException + throw Volatile.Read(ref _state.ColdState)?.TerminalException ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); } @@ -615,13 +696,13 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask ReadNextPublishedResultAsync() { - CommandResult? result = _current; - while (_commandIndex < _commands.Count) + CommandResult? result = _state.Current; + while (_state.CommandIndex < _state.Commands.Count) { - result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); - _current = result; - _currentPublished = false; - if (!_commands.ItemRef(_commandIndex).SuppressEnumeration) + result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); + _state.Current = result; + _state.CurrentPublished = false; + if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) return result; var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); @@ -636,34 +717,34 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) (cold.DrainErrors ??= new()).Add(exception); if (completeError is { TransactionStatus: TransactionStatus.Unknown }) await SkipDiscardedCommandsAsync().ConfigureAwait(false); - _commandIndex++; - _current = null; - Interlocked.Exchange(ref _phase, PhaseDraining); + _state.CommandIndex++; + _state.Current = null; + Interlocked.Exchange(ref _state.Phase, PhaseDraining); NotifyDrainStarted(); - _consumerDetached = true; + _state.ConsumerDetached = true; await DrainAsync().ConfigureAwait(false); throw exception; } - _commandIndex++; + _state.CommandIndex++; } if (result is null) throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); await CompleteBatchAsync().ConfigureAwait(false); - _consumerObservedCompletion = true; + _state.ConsumerObservedCompletion = true; return null; } CommandResult? ReadNextPublishedResult() { - CommandResult? result = _current; - while (_commandIndex < _commands.Count) + CommandResult? result = _state.Current; + while (_state.CommandIndex < _state.Commands.Count) { - result = ReadResult(_commandIndex); - _current = result; - _currentPublished = false; - if (!_commands.ItemRef(_commandIndex).SuppressEnumeration) + result = ReadResult(_state.CommandIndex); + _state.Current = result; + _state.CurrentPublished = false; + if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) return result; var completeError = CompleteCurrentResult(); @@ -678,36 +759,36 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) (cold.DrainErrors ??= new()).Add(exception); if (completeError is { TransactionStatus: TransactionStatus.Unknown }) SkipDiscardedCommands(); - _commandIndex++; - _current = null; - Interlocked.Exchange(ref _phase, PhaseDraining); + _state.CommandIndex++; + _state.Current = null; + Interlocked.Exchange(ref _state.Phase, PhaseDraining); NotifyDrainStarted(); - _consumerDetached = true; + _state.ConsumerDetached = true; Drain(); throw exception; } - _commandIndex++; + _state.CommandIndex++; } if (result is null) throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); CompleteBatch(); - _consumerObservedCompletion = true; + _state.ConsumerObservedCompletion = true; return null; } // Reads through the command's execute prelude and initializes the protocol-static result. async ValueTask ReadResultAsync(int commandIndex) { - var context = _context; + var context = _state.Context; var decoder = context.Decoder; // After close, a fresh command must not consume bytes left by its predecessor. if (context.IsProtocolClosed) throw context.FlowTerminationException; PgError? error; RowDescription? requestedRowDescription; - ref readonly var command = ref _commands.ItemRef(commandIndex); + ref readonly var command = ref _state.Commands.ItemRef(commandIndex); var describeOnly = command.DescribeOnly; var hasPreparedDescription = command.Descriptor is { IsPrepared: true, PreparedRowDescription: not null }; @@ -761,11 +842,11 @@ async ValueTask ReadResultAsync(int commandIndex) CommandResult ReadResult(int commandIndex) { - var context = _context; + var context = _state.Context; var decoder = context.Decoder; if (context.IsProtocolClosed) throw context.FlowTerminationException; - ref readonly var command = ref _commands.ItemRef(commandIndex); + ref readonly var command = ref _state.Commands.ItemRef(commandIndex); decoder.UseReadTimeout(command.Timeout); PgError? error; RowDescription? requestedRowDescription; @@ -791,9 +872,9 @@ CommandResult InitializeResult( int commandIndex, PgError? error, RowDescription? requestedRowDescription, ParameterTypeList? preparationParameterTypes = null) { - var context = _context; + var context = _state.Context; ref readonly var readState = ref context.GetProtocolStatic(); - ref readonly var command = ref _commands.ItemRef(commandIndex); + ref readonly var command = ref _state.Commands.ItemRef(commandIndex); readState.ResultMessageEnumerator.Initialize(command, context.Decoder); var result = readState.CommandResult; var descriptor = command.Descriptor; @@ -805,59 +886,59 @@ CommandResult InitializeResult( preparationParameterTypes ?? descriptor.ParameterTypes, requestedRowDescription?.Preserve()); } - result.Initialize(this, commandIndex, descriptor, requestedRowDescription, + result.Initialize(_ops.Flow, commandIndex, descriptor, requestedRowDescription, !command.DescribeOnly, command.IsSimple(), error); - _commandObserver?.OnCommandResult(this, result, _commandObserverState); + _ops.OnCommandResult(result); return result; } async ValueTask<(PgError Error, TransactionStatus TransactionStatus)?> CompleteCurrentResultAsync() { - var enumerator = _context.GetProtocolStatic().ResultMessageEnumerator; + var enumerator = _state.Context.GetProtocolStatic().ResultMessageEnumerator; await enumerator.DisposeAsync().ConfigureAwait(false); return enumerator.CompleteError; } (PgError Error, TransactionStatus TransactionStatus)? CompleteCurrentResult() { - var enumerator = _context.GetProtocolStatic().ResultMessageEnumerator; + var enumerator = _state.Context.GetProtocolStatic().ResultMessageEnumerator; enumerator.Dispose(); return enumerator.CompleteError; } async ValueTask SkipDiscardedCommandsAsync() { - while (++_commandIndex < _commands.Count && !_commands[_commandIndex].WithSync) { } + while (++_state.CommandIndex < _state.Commands.Count && !_state.Commands[_state.CommandIndex].WithSync) { } await ReadRfqAsync().ConfigureAwait(false); - if (_commandIndex == _commands.Count) - _readFlowRfq = false; + if (_state.CommandIndex == _state.Commands.Count) + _state.ReadFlowRfq = false; } void SkipDiscardedCommands() { - while (++_commandIndex < _commands.Count && !_commands[_commandIndex].WithSync) { } + while (++_state.CommandIndex < _state.Commands.Count && !_state.Commands[_state.CommandIndex].WithSync) { } ReadRfq(); - if (_commandIndex == _commands.Count) - _readFlowRfq = false; + if (_state.CommandIndex == _state.Commands.Count) + _state.ReadFlowRfq = false; } async ValueTask ReadRfqAsync() { - var message = await _context.Decoder.GetNextAsync().ConfigureAwait(false); + var message = await _state.Context.Decoder.GetNextAsync().ConfigureAwait(false); if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) PgErrorException.Throw(rfqError); } void ReadRfq() { - var message = _context.Decoder.GetNext(); + var message = _state.Context.Decoder.GetNext(); if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) PgErrorException.Throw(rfqError); } async ValueTask CompleteBatchAsync() { - if (_readFlowRfq) + if (_state.ReadFlowRfq) await ReadRfqAsync().ConfigureAwait(false); await DisposeRegistrationsAsync().ConfigureAwait(false); Finish(); @@ -865,7 +946,7 @@ async ValueTask CompleteBatchAsync() void CompleteBatch() { - if (_readFlowRfq) + if (_state.ReadFlowRfq) ReadRfq(); DisposeRegistrations(); Finish(); @@ -875,13 +956,13 @@ void CompleteBatch() // consumer must observe, then complete the pipeline task. void Finish() { - _context.GetProtocolStatic().Reset(); - _current = null; - _currentPublished = false; + _state.Context.GetProtocolStatic().Reset(); + _state.Current = null; + _state.CurrentPublished = false; if (IsCancelRequested) Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, - new OperationCanceledException(_coldState!.DeliverToken), null); - else if (Volatile.Read(ref _coldState)?.CloseException is { } close) + new OperationCanceledException(_state.ColdState!.DeliverToken), null); + else if (Volatile.Read(ref _state.ColdState)?.CloseException is { } close) Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, close, null); CompletePipelineTask(null); } @@ -894,13 +975,13 @@ bool IsOwnCancellation(PgError error) void FaultFromOwner(Exception exception) { Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - if (Volatile.Read(ref _phase) == PhaseCompleted) + if (Volatile.Read(ref _state.Phase) == PhaseCompleted) return; DisposeRegistrations(); - _current = null; - _currentPublished = false; + _state.Current = null; + _state.CurrentPublished = false; if (HasDecoder) - _context.GetProtocolStatic().Reset(); + _state.Context.GetProtocolStatic().Reset(); CompletePipelineTask(exception); } @@ -910,24 +991,26 @@ bool TryTakeOverDrain() { while (true) { - var phase = Volatile.Read(ref _phase); + var phase = Volatile.Read(ref _state.Phase); if (phase is PhaseInitial && !HasDecoder) return false; if (phase is not (PhaseInitial or PhaseResultReady)) return false; - if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) continue; NotifyDrainStarted(); - _consumerDetached = true; - ThreadPool.UnsafeQueueUserWorkItem(static state => _ = ((CommandExecutionFlow)state!).DrainAsync(), this); + _state.ConsumerDetached = true; + ThreadPool.UnsafeQueueUserWorkItem(static state => + _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ops.Flow); return true; } } void NotifyDrainStarted() { - if (Interlocked.Exchange(ref _drainStarted, 1) is 0) - _commandObserver?.OnDrainStarted(this, _commandObserverState); + if (Interlocked.Exchange(ref _state.DrainStarted, 1) is 0) + _ops.OnDrainStarted(); } // Autonomous drain. Owns the decoder until the pipeline task completes. Never throws. @@ -935,30 +1018,30 @@ async ValueTask DrainAsync() { try { - var result = _current; + var result = _state.Current; if (result is null) { - await new ValueTask(this, _readySource.Version).ConfigureAwait(false); - if (_commandIndex < 0) - _commandIndex = 0; - if (_commandIndex >= _commands.Count) + await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); + if (_state.CommandIndex < 0) + _state.CommandIndex = 0; + if (_state.CommandIndex >= _state.Commands.Count) { await CompleteBatchAsync().ConfigureAwait(false); return; } - result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); } while (true) { var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); CaptureDrainError(result, completeError); - _currentPublished = false; + _state.CurrentPublished = false; if (completeError is { TransactionStatus: TransactionStatus.Unknown }) await SkipDiscardedCommandsAsync().ConfigureAwait(false); - if (++_commandIndex >= _commands.Count) + if (++_state.CommandIndex >= _state.Commands.Count) break; - result = await ReadResultAsync(_commandIndex).ConfigureAwait(false); + result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); } await CompleteBatchAsync().ConfigureAwait(false); } @@ -966,7 +1049,7 @@ async ValueTask DrainAsync() { // A timeout during semantic drain escalates the same cancellation episode immediately. // The pipeline failure then hands any remaining wire obligation to recovery. - RequestCancel(default, CancellationScope.RemainingFlow, + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); FaultFromOwner(ex); } @@ -983,45 +1066,46 @@ void HandleReadTimeout(TimeoutException exception) { Interlocked.CompareExchange( ref GetOrCreateColdState().TerminalException, exception, null); - _consumerDetached = true; + _state.ConsumerDetached = true; NotifyDrainStarted(); - Interlocked.Exchange(ref _phase, PhaseDraining); - RequestCancel(default, CancellationScope.RemainingFlow, + Interlocked.Exchange(ref _state.Phase, PhaseDraining); + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); - ThreadPool.UnsafeQueueUserWorkItem( - static state => _ = ((CommandExecutionFlow)state!).DrainAsync(), this); + ThreadPool.UnsafeQueueUserWorkItem(static state => + _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ops.Flow); } void Drain() { try { - var result = _current; + var result = _state.Current; if (result is null) { // Synchronous disposal before any read. The activation bridge normally completed long // ago, so this bridge is rarely more than a status check. - new ValueTask(this, _readySource.Version).AsTask().GetAwaiter().GetResult(); - if (_commandIndex < 0) - _commandIndex = 0; - if (_commandIndex >= _commands.Count) + new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).AsTask().GetAwaiter().GetResult(); + if (_state.CommandIndex < 0) + _state.CommandIndex = 0; + if (_state.CommandIndex >= _state.Commands.Count) { CompleteBatch(); return; } - result = ReadResult(_commandIndex); + result = ReadResult(_state.CommandIndex); } while (true) { var completeError = CompleteCurrentResult(); CaptureDrainError(result, completeError); - _currentPublished = false; + _state.CurrentPublished = false; if (completeError is { TransactionStatus: TransactionStatus.Unknown }) SkipDiscardedCommands(); - if (++_commandIndex >= _commands.Count) + if (++_state.CommandIndex >= _state.Commands.Count) break; - result = ReadResult(_commandIndex); + result = ReadResult(_state.CommandIndex); } CompleteBatch(); } @@ -1031,30 +1115,30 @@ void Drain() } } - ValueTask DisposeAsync() + internal ValueTask DisposeAsync() { while (true) { - var phase = Volatile.Read(ref _phase); + var phase = Volatile.Read(ref _state.Phase); switch (phase) { case PhaseInitial: case PhaseResultReady: - if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) continue; NotifyDrainStarted(); - _consumerDetached = true; - if (_current is { IsComplete: false } - || _commandIndex + 1 < _commands.Count) - RequestCancel(default, CancellationScope.RemainingFlow); - return WaitForDrainOnDispose ? DisposeDrainAsync() : FireAndForgetDrain(); + _state.ConsumerDetached = true; + if (_state.Current is { IsComplete: false } + || _state.CommandIndex + 1 < _state.Commands.Count) + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + return _state.WaitForDrainOnDispose ? DisposeDrainAsync() : FireAndForgetDrain(); case PhaseReading: - _consumerDetached = true; + _state.ConsumerDetached = true; NotifyDrainStarted(); - RequestCancel(default, CancellationScope.RemainingFlow); - return WaitForDrainOnDispose ? DisposeCompletedAsync() : default; + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + return _state.WaitForDrainOnDispose ? DisposeCompletedAsync() : default; default: - return !WaitForDrainOnDispose || _consumerObservedCompletion + return !_state.WaitForDrainOnDispose || _state.ConsumerObservedCompletion ? default : DisposeCompletedAsync(); } @@ -1086,44 +1170,44 @@ async ValueTask WaitForCompletionAsync() { try { - await WaitForComplete().ConfigureAwait(false); + await _ops.Flow.WaitForComplete().ConfigureAwait(false); } catch (PgClientClosedException) { } } - void Dispose() + internal void Dispose() { - if (!IsAsyncAtDispatch) + if (!_ops.IsAsyncAtDispatch) EnsureSyncHandoff(); while (true) { - var phase = Volatile.Read(ref _phase); + var phase = Volatile.Read(ref _state.Phase); switch (phase) { case PhaseInitial: case PhaseResultReady: - if (Interlocked.CompareExchange(ref _phase, PhaseDraining, phase) != phase) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) continue; NotifyDrainStarted(); - _consumerDetached = true; - if (_current is { IsComplete: false } - || _commandIndex + 1 < _commands.Count) - RequestCancel(default, CancellationScope.RemainingFlow); + _state.ConsumerDetached = true; + if (_state.Current is { IsComplete: false } + || _state.CommandIndex + 1 < _state.Commands.Count) + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); Drain(); - if (WaitForDrainOnDispose) + if (_state.WaitForDrainOnDispose) DisposeCompleted(); return; case PhaseReading: - _consumerDetached = true; + _state.ConsumerDetached = true; NotifyDrainStarted(); - RequestCancel(default, CancellationScope.RemainingFlow); - if (WaitForDrainOnDispose) + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + if (_state.WaitForDrainOnDispose) DisposeCompleted(); return; default: - if (WaitForDrainOnDispose && !_consumerObservedCompletion) + if (_state.WaitForDrainOnDispose && !_state.ConsumerObservedCompletion) DisposeCompleted(); return; } @@ -1134,7 +1218,7 @@ void DisposeCompleted() { try { - WaitForCompleteSynchronously(); + _ops.Flow.WaitForCompleteSynchronously(); } catch (PgClientClosedException) { @@ -1145,7 +1229,7 @@ void DisposeCompleted() void CaptureDrainError(CommandResult result, (PgError Error, TransactionStatus TransactionStatus)? completeError) { - if (!_consumerDetached || _currentPublished) + if (!_state.ConsumerDetached || _state.CurrentPublished) return; var error = result.Error ?? completeError?.Error; if (error is null || IsOwnCancellation(error)) @@ -1156,7 +1240,7 @@ void CaptureDrainError(CommandResult result, void ThrowDrainErrors() { - if (Volatile.Read(ref _coldState)?.DrainErrors is not { Count: > 0 } errors) + if (Volatile.Read(ref _state.ColdState)?.DrainErrors is not { Count: > 0 } errors) return; if (errors.Count is 1) ExceptionDispatchInfo.Throw(errors[0]); @@ -1165,13 +1249,11 @@ void ThrowDrainErrors() // When true, disposal waits for the drain to reach RFQ and for framework release. Otherwise it // returns while the drain continues autonomously. - internal bool WaitForDrainOnDispose { get; set; } = true; - void RegisterCancellation(CancellationToken callerToken) { // Keep the second token/registration pair off ordinary flow objects. Default-token traffic // does not need cancellation state at all and remains the allocation/footprint hot path. - var cancellation = Volatile.Read(ref _coldState); + var cancellation = Volatile.Read(ref _state.ColdState); if (callerToken.CanBeCanceled || cancellation is not null) { cancellation ??= GetOrCreateColdState(); @@ -1184,26 +1266,26 @@ void RegisterCancellation(CancellationToken callerToken) } if (callerToken.CanBeCanceled && cancellation.CallerRegistration == default) cancellation.CallerRegistration = callerToken.UnsafeRegister(static (state, token) - => ((CommandExecutionFlow)state!).RequestCancel( - token, CancellationScope.CurrentWindow), this); + => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( + token, CommandExecutionCancellationScope.CurrentWindow), _ops.Flow); } - if (_flowToken.CanBeCanceled && _flowRegistration == default) - _flowRegistration = _flowToken.UnsafeRegister(static (state, token) - => ((CommandExecutionFlow)state!).RequestCancel( - token, CancellationScope.RemainingFlow), this); + if (_state.FlowToken.CanBeCanceled && _state.FlowRegistration == default) + _state.FlowRegistration = _state.FlowToken.UnsafeRegister(static (state, token) + => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( + token, CommandExecutionCancellationScope.RemainingFlow), _ops.Flow); } ValueTask DisposeRegistrationsAsync() { - var cancellation = Volatile.Read(ref _coldState); + var cancellation = Volatile.Read(ref _state.ColdState); var callerRegistration = cancellation?.CallerRegistration ?? default; - if (callerRegistration == default && _flowRegistration == default) + if (callerRegistration == default && _state.FlowRegistration == default) return default; if (cancellation is not null) cancellation.CallerRegistration = default; - var flowRegistration = _flowRegistration; - _flowRegistration = default; + var flowRegistration = _state.FlowRegistration; + _state.FlowRegistration = default; return DisposeRegistrationsAsync(callerRegistration, flowRegistration); static async ValueTask DisposeRegistrationsAsync( @@ -1217,23 +1299,23 @@ static async ValueTask DisposeRegistrationsAsync( void DisposeRegistrations() { - var cancellation = Volatile.Read(ref _coldState); + var cancellation = Volatile.Read(ref _state.ColdState); var callerRegistration = cancellation?.CallerRegistration ?? default; if (cancellation is not null) cancellation.CallerRegistration = default; - var flowRegistration = _flowRegistration; - _flowRegistration = default; + var flowRegistration = _state.FlowRegistration; + _state.FlowRegistration = default; callerRegistration.Dispose(); flowRegistration.Dispose(); } // Cancellation only latches intent and requests a backend cancel. The frame owning the decoder // delivers it after the wire is back at RFQ. An idle flow drains autonomously first. - void RequestCancel(CancellationToken token, CancellationScope scope, + void RequestCancel(CancellationToken token, CommandExecutionCancellationScope scope, BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, BackendCancellationTiming subsequentTiming = BackendCancellationTiming.AfterGrace) { - if (Volatile.Read(ref _phase) == PhaseCompleted) + if (Volatile.Read(ref _state.Phase) == PhaseCompleted) return; var cancellation = GetOrCreateColdState(); cancellation.DeliverToken = token; @@ -1246,7 +1328,7 @@ void RequestCancel(CancellationToken token, CancellationScope scope, TryTakeOverDrain(); } - static void RaiseCancellationScope(ColdState cancellation, CancellationScope scope) + static void RaiseCancellationScope(CommandExecutionColdState cancellation, CommandExecutionCancellationScope scope) { var requested = (int)scope; var current = Volatile.Read(ref cancellation.Scope); @@ -1281,13 +1363,13 @@ internal Task CancelAsync() ?? Interlocked.CompareExchange(ref cancellation.Delivery, new(TaskCreationOptions.RunContinuationsAsynchronously), null) ?? cancellation.Delivery; - if (Volatile.Read(ref _phase) is PhaseCompleted) + if (Volatile.Read(ref _state.Phase) is PhaseCompleted) { delivery.TrySetResult(); return delivery.Task; } - RequestCancel(default, CancellationScope.RemainingFlow); - if (Volatile.Read(ref _phase) is PhaseCompleted) + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + if (Volatile.Read(ref _state.Phase) is PhaseCompleted) delivery.TrySetResult(); return delivery.Task; } @@ -1302,22 +1384,18 @@ void RequestBackendCancellation() episodeKey = Interlocked.CompareExchange( ref cancellation.EpisodeKey, created, null) ?? created; } - _context.RequestBackendCancellation( - this, CancellationWindow, + _state.Context.RequestBackendCancellation( + _ops.Flow, _ops.Flow.CancellationWindow, (BackendCancellationTiming)Volatile.Read(ref cancellation.Timing), Volatile.Read(ref cancellation.Delivery), episodeKey, - Math.Max(Volatile.Read(ref cancellation.Scope), (int)CancellationScope.CurrentWindow), + Math.Max(Volatile.Read(ref cancellation.Scope), (int)CommandExecutionCancellationScope.CurrentWindow), (BackendCancellationTiming)Volatile.Read(ref cancellation.SubsequentTiming)); } - // Finish and FaultFromOwner reset the shared read objects before the pipeline task completes, and - // Current is null once the consumer observed the terminal, so nothing outlives the flow. - internal override bool ResetsSharedReadStateBeforeRelease => true; - // Graceful stop. An unactivated flow releases its consumer, the closing wire owns its response. // An idle activated flow drains itself to RFQ so the pipeline can complete. - protected override void OnStopping(Exception exception) + internal void OnStopping(Exception exception) { Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); if (CompleteReady(exception, runContinuationsAsynchronously: true)) @@ -1330,7 +1408,7 @@ protected override void OnStopping(Exception exception) // Forceful abort. No frame can read a dead wire, so an idle owner faults the pipeline task // directly. A frame in flight fails on its own read. - protected override void OnAbort(Exception exception) + internal void OnAbort(Exception exception) { Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); if (CompleteReady(exception, runContinuationsAsynchronously: true)) @@ -1340,67 +1418,72 @@ protected override void OnAbort(Exception exception) } while (true) { - var phase = Volatile.Read(ref _phase); + var phase = Volatile.Read(ref _state.Phase); if (phase is not (PhaseInitial or PhaseResultReady)) return; - if (Interlocked.CompareExchange(ref _phase, PhaseCompleted, phase) != phase) + if (Interlocked.CompareExchange(ref _state.Phase, PhaseCompleted, phase) != phase) continue; Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - _pipelineTaskSource.SetException(exception, runContinuationsAsynchronously: true); + _state.PipelineTaskSource.SetException(exception, runContinuationsAsynchronously: true); return; } } - internal override void Fail(Exception exception) + internal void Fail(Exception exception) { // A result callback failed on the frame that owns the decoder. Its throw propagates there. Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); } - protected override void OnReleasing(Exception? exception) + internal void OnReleasing(Exception? exception) { - Volatile.Read(ref _coldState)?.Delivery?.TrySetResult(); + Volatile.Read(ref _state.ColdState)?.Delivery?.TrySetResult(); DisposeRegistrations(); - _commands.Return(); + _state.Commands.Return(); } - protected override void OnDiscarded() + internal void OnDiscarded() { - GetObserver(out var observerState)?.OnCompleting(this, null, observerState); - _commands.Return(); + _ops.OnDiscarded(); + _state.Commands.Return(); } - protected override void OnReset() + internal void OnReset() { - _phase = PhaseInitial; - _commandIndex = -1; - _context = default; - _current = null; - _currentPublished = false; - _readFlowRfq = false; - _consumerDetached = false; - _consumerObservedCompletion = false; - _readySource.Reset(); - _pipelineTaskSource.Reset(); - _readyCompletion = 0; - _drainStarted = 0; - _flowToken = default; - _flowRegistration = default; - _coldState = null; - _syncHandoffClaimed = false; - _handoffEvent?.ResetInteraction(); - WaitForDrainOnDispose = true; + _state.Phase = PhaseInitial; + _state.CommandIndex = -1; + _state.Context = default; + _state.ContextPublished = false; + _state.Current = null; + _state.CurrentPublished = false; + _state.ReadFlowRfq = false; + _state.ConsumerDetached = false; + _state.ConsumerObservedCompletion = false; + _state.ReadySource.Reset(); + _state.PipelineTaskSource.Reset(); + _state.ReadyCompletion = 0; + _state.DrainStarted = 0; + _state.FlowToken = default; + _state.FlowRegistration = default; + _state.ColdState = null; + _state.SyncHandoffClaimed = false; + _state.HandoffEvent?.ResetInteraction(); + _state.WaitForDrainOnDispose = true; } - bool IValueTaskSource.GetResult(short token) => _readySource.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _readySource.GetStatus(token); +} + +internal sealed partial class CommandExecutionFlow +{ + bool IValueTaskSource.GetResult(short token) => _state.ReadySource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _state.ReadySource.GetStatus(token); void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - => _readySource.OnCompleted(continuation, state, token, flags); + => _state.ReadySource.OnCompleted(continuation, state, token, flags); - void IValueTaskSource.GetResult(short token) => _pipelineTaskSource.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _pipelineTaskSource.GetStatus(token); + void IValueTaskSource.GetResult(short token) => _state.PipelineTaskSource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _state.PipelineTaskSource.GetStatus(token); void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - => _pipelineTaskSource.OnCompleted(continuation, state, token, flags); + => _state.PipelineTaskSource.OnCompleted(continuation, state, token, flags); public readonly struct Enumerator : IAsyncEnumerator, IDisposable { @@ -1421,17 +1504,17 @@ internal Enumerator(CommandExecutionFlow flow, CancellationToken cancellationTok public Enumerator GetEnumerator() => this; - public bool MoveNext() => _flow?.MoveNext() ?? false; + public bool MoveNext() => _flow?.Core.MoveNext() ?? false; public ValueTask MoveNextAsync() => MoveNextAsync(_cancellationToken); public ValueTask MoveNextAsync(CancellationToken cancellationToken) - => _flow is null ? new(false) : _flow.MoveNextAsync(cancellationToken); + => _flow is null ? new(false) : _flow.Core.MoveNextAsync(cancellationToken); - public CommandResult Current => _flow?._current ?? default!; + public CommandResult Current => _flow?._state.Current ?? default!; - public ValueTask DisposeAsync() => _flow is null ? default : _flow.DisposeAsync(); + public ValueTask DisposeAsync() => _flow is null ? default : _flow.Core.DisposeAsync(); - public void Dispose() => _flow?.Dispose(); + public void Dispose() => _flow?.Core.Dispose(); } } diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index d4c6406..28e0c0f 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -527,7 +527,7 @@ void IValueTaskSource.OnCompleted(Action continuation, => _activationTaskSource.OnCompleted(continuation, state, token, flags); - protected readonly struct Context + protected internal readonly struct Context { readonly ExecutionControl _executionControl; internal Context(ExecutionControl executionControl) @@ -600,7 +600,7 @@ public DecoderAwaitable GetDecoderAuto(CancellationToken cancellationToken = def // compiler checks IsCompleted and only schedules via (Unsafe)OnCompleted(Action) when not ready. // Direct dispatchers (CommandFlow's shared-promise pattern) instead use IsCompleted + // (Unsafe)OnCompleted(Action, object?) to register without a closure allocation. - protected readonly struct DecoderAwaitable : ICriticalNotifyCompletion + protected internal readonly struct DecoderAwaitable : ICriticalNotifyCompletion { readonly ExecutionControl control; readonly CancellationToken cancellationToken; @@ -675,7 +675,7 @@ public void UnsafeOnCompleted(Action continuation, object? state) // The ConfigureAwait(false) variant: skips scheduling-context capture. Action overloads are // for the C# `await` syntax (compiler calls UnsafeOnCompleted on ICriticalNotifyCompletion). - protected readonly struct ConfiguredDecoderAwaitable : ICriticalNotifyCompletion + protected internal readonly struct ConfiguredDecoderAwaitable : ICriticalNotifyCompletion { readonly ExecutionControl control; readonly CancellationToken cancellationToken; From ecf730bba9721fde6056d2bdd62490c4016c71da Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 01:36:06 +0200 Subject: [PATCH 088/136] Run ADO commands over the shared execution core --- Slon.Tests/Ado/ReaderDisposalTests.cs | 10 +- Slon/Ado/AdoBatchCore.Preparation.cs | 4 +- Slon/Ado/AdoBatchCore.cs | 44 +++--- Slon/Ado/AdoCommandFlow.cs | 187 ++++++++++++++++++-------- Slon/Ado/AdoConnectionProxy.cs | 2 +- Slon/SlonBatch.cs | 21 ++- Slon/SlonCommand.cs | 21 ++- Slon/SlonDataReader.cs | 14 +- Slon/SlonDataSource.cs | 8 +- 9 files changed, 212 insertions(+), 99 deletions(-) diff --git a/Slon.Tests/Ado/ReaderDisposalTests.cs b/Slon.Tests/Ado/ReaderDisposalTests.cs index 6a3ba5b..5ca5f6b 100644 --- a/Slon.Tests/Ado/ReaderDisposalTests.cs +++ b/Slon.Tests/Ado/ReaderDisposalTests.cs @@ -97,17 +97,17 @@ public async Task HiddenLaterResult_DisposalCreatesCancellationState(bool async) Assert.IsTrue(HasCancellationState(flow)); } - static CommandFlow GetFlow(SlonDataReader reader) + static AdoCommandExecutionFlow GetFlow(SlonDataReader reader) { var enumerator = typeof(SlonDataReader).GetField("_enumerator", AllInstanceFields)! .GetValue(reader)!; var flowField = enumerator.GetType().GetFields(AllInstanceFields) - .Single(static field => field.FieldType == typeof(CommandFlow)); - return (CommandFlow)flowField.GetValue(enumerator)!; + .Single(static field => field.FieldType == typeof(AdoCommandExecutionFlow)); + return (AdoCommandExecutionFlow)flowField.GetValue(enumerator)!; } - static bool HasCancellationState(CommandFlow flow) - => FindField(flow.GetType(), "_cancellationState").GetValue(flow) is not null; + static bool HasCancellationState(AdoCommandExecutionFlow flow) + => flow.HasCancellationState; static FieldInfo FindField(Type type, string name) { diff --git a/Slon/Ado/AdoBatchCore.Preparation.cs b/Slon/Ado/AdoBatchCore.Preparation.cs index 76ebe34..d5285d6 100644 --- a/Slon/Ado/AdoBatchCore.Preparation.cs +++ b/Slon/Ado/AdoBatchCore.Preparation.cs @@ -25,7 +25,7 @@ public void Prepare(DbParameterCollection? parameters) void PrepareCore(DbParameterCollection? parameters) { var operation = Preparation.Begin(_fieldRef); - CommandFlow.Enumerator enumerator = default; + AdoCommandExecutionFlow.Enumerator enumerator = default; try { var flow = Enqueue(parameters, CommandBehavior.SchemaOnly, GetDependencies(), @@ -85,7 +85,7 @@ static async ValueTask PrepareAsyncCore(FieldRef> fieldRe DbParameterCollection? parameters, CancellationToken cancellationToken) { var operation = Preparation.Begin(fieldRef); - CommandFlow.Enumerator enumerator = default; + AdoCommandExecutionFlow.Enumerator enumerator = default; try { var dependencies = await fieldRef.Invoke().GetDependenciesAsync(cancellationToken) diff --git a/Slon/Ado/AdoBatchCore.cs b/Slon/Ado/AdoBatchCore.cs index 7fac606..665549e 100644 --- a/Slon/Ado/AdoBatchCore.cs +++ b/Slon/Ado/AdoBatchCore.cs @@ -20,7 +20,7 @@ partial struct AdoBatchCore where TCommand : IAdoCommand TimeSpan _timeout; TimeSpan? _pendingTimeout; bool _enableErrorBarriers; - CommandFlow? _activeFlow; + AdoCommandExecutionFlow? _activeFlow; AdoCommandList _commands; public AdoBatchCore(FieldRef> fieldRef) @@ -194,7 +194,7 @@ SlonDataSource.PgDbDependencies GetDependencies() return (dataSource ?? connection!.DbDataSource).GetDbDependenciesAsync(cancellationToken); } - CommandFlow Enqueue(DbParameterCollection? parameters, CommandBehavior behavior, + AdoCommandExecutionFlow Enqueue(DbParameterCollection? parameters, CommandBehavior behavior, SlonDataSource.PgDbDependencies dependencies, bool preparing = false) { if (TryGetDataSource(out var dataSource, out var connection)) @@ -202,20 +202,24 @@ CommandFlow Enqueue(DbParameterCollection? parameters, CommandBehavior behavior, ThrowIfHasCloseConnection(behavior); var pendingTimeout = PendingTimeout; return dataSource.EnqueueCommands( - new AdoCommandFlow( - async: false, _fieldRef, parameters, behavior, dependencies, + new AdoCommandExecutionFlow( + async: false, (IAdoCommandExecutionOwner)_fieldRef.Instance, + parameters, behavior, dependencies, connection: null, pendingTimeout, preparing, _commands.Count, - _explicitlyPrepared && _fieldRef.Instance is SlonCommand ? null : _fieldRef.Instance), + _explicitlyPrepared && _fieldRef.Instance is SlonCommand + ? null + : (IAdoCommandExecutionOwner)_fieldRef.Instance), pendingTimeout); } connection ??= ThrowConnectionNotInitialized(); - return connection.Enqueue(new AdoCommandFlow( - async: false, _fieldRef, parameters, behavior, dependencies, - connection, PendingTimeout, preparing, _commands.Count, _fieldRef.Instance)); + return connection.Enqueue(new AdoCommandExecutionFlow( + async: false, (IAdoCommandExecutionOwner)_fieldRef.Instance, + parameters, behavior, dependencies, connection, PendingTimeout, preparing, + _commands.Count, (IAdoCommandExecutionOwner)_fieldRef.Instance)); } - ValueTask EnqueueAsync(DbParameterCollection? parameters, + ValueTask EnqueueAsync(DbParameterCollection? parameters, CommandBehavior behavior, SlonDataSource.PgDbDependencies dependencies, CancellationToken cancellationToken, bool preparing = false) { @@ -224,17 +228,21 @@ ValueTask EnqueueAsync(DbParameterCollection? parameters, ThrowIfHasCloseConnection(behavior); var pendingTimeout = PendingTimeout; return dataSource.EnqueueCommandsAsync( - new AdoCommandFlow( - async: true, _fieldRef, parameters, behavior, dependencies, + new AdoCommandExecutionFlow( + async: true, (IAdoCommandExecutionOwner)_fieldRef.Instance, + parameters, behavior, dependencies, connection: null, pendingTimeout, preparing, _commands.Count, - _explicitlyPrepared && _fieldRef.Instance is SlonCommand ? null : _fieldRef.Instance), + _explicitlyPrepared && _fieldRef.Instance is SlonCommand + ? null + : (IAdoCommandExecutionOwner)_fieldRef.Instance), pendingTimeout, cancellationToken); } connection ??= ThrowConnectionNotInitialized(); - return connection.EnqueueAsync(new AdoCommandFlow( - async: true, _fieldRef, parameters, behavior, dependencies, - connection, PendingTimeout, preparing, _commands.Count, _fieldRef.Instance), cancellationToken); + return connection.EnqueueAsync(new AdoCommandExecutionFlow( + async: true, (IAdoCommandExecutionOwner)_fieldRef.Instance, + parameters, behavior, dependencies, connection, PendingTimeout, preparing, + _commands.Count, (IAdoCommandExecutionOwner)_fieldRef.Instance), cancellationToken); } [DoesNotReturn] @@ -346,7 +354,7 @@ public ValueTask ExecuteNonQueryAsync(DbParameterCollection? parameters, Ca { ref var thisRef = ref fieldRef.Invoke(); using var activity = thisRef.StartActivity(); - CommandFlow.Enumerator enumerator = default; + AdoCommandExecutionFlow.Enumerator enumerator = default; try { thisRef.ThrowIfDisposed(); @@ -607,9 +615,9 @@ static void ThrowPreparedCancellationNotSupported() => throw new NotSupportedException( "A datasource-prepared command can have multiple executions; cancel an execution with its token."); - internal void OnFlowStarted(CommandFlow flow) => Volatile.Write(ref _activeFlow, flow); + internal void OnFlowStarted(AdoCommandExecutionFlow flow) => Volatile.Write(ref _activeFlow, flow); - internal void OnFlowCompleting(CommandFlow flow, Exception? exception) + internal void OnFlowCompleting(AdoCommandExecutionFlow flow, Exception? exception) { // A flow-level fault while holding an ADO connection lease breaks that lease. SQL errors don't // reach here: they surface on CommandResult and the flow completes cleanly. OnCompleting runs diff --git a/Slon/Ado/AdoCommandFlow.cs b/Slon/Ado/AdoCommandFlow.cs index 39ee129..626f9bc 100644 --- a/Slon/Ado/AdoCommandFlow.cs +++ b/Slon/Ado/AdoCommandFlow.cs @@ -1,5 +1,6 @@ using System.Data; using System.Data.Common; +using System.Threading.Tasks.Sources; using Slon.Pg; using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; @@ -15,40 +16,23 @@ readonly struct AdoCommandFlowOptions internal object? ResultObserverState { get; init; } } -sealed class AdoCommandFlowObserver : CommandFlowObserver - where TCommand : IAdoCommand +interface IAdoCommandExecutionOwner { - internal static readonly AdoCommandFlowObserver Instance = new(); - - protected internal override void OnStarted(CommandFlow flow, object? state) - { - switch (((AdoCommandFlow)flow).LifetimeOwner) - { - case SlonCommand command: - command.OnFlowStarted(flow); - break; - case SlonBatch batch: - batch.OnFlowStarted(flow); - break; - } - } - - protected internal override void OnCommandResult(CommandFlow flow, CommandResult result, object? state) - => ((AdoCommandFlow)flow).ObserveResult(result); + AdoCommandFlowOptions CreateExecutionOptions( + DbParameterCollection? parameters, CommandBehavior behavior, + SlonDataSource.PgDbDependencies dependencies, SlonConnection? connection, + PgConnection pgConnection, TimeSpan? pendingTimeout, bool preparing); + void OnFlowStarted(AdoCommandExecutionFlow flow); + void OnFlowCompleting(AdoCommandExecutionFlow flow, Exception? exception); +} - protected internal override void OnCompleting(PgClientFlow flow, Exception? exception, object? state) - { - switch (((AdoCommandFlow)flow).LifetimeOwner) - { - case SlonCommand command: - command.OnFlowCompleting((CommandFlow)flow, exception); - break; - case SlonBatch batch: - batch.OnFlowCompleting((CommandFlow)flow, exception); - break; - } - } +sealed class AdoCommandExecutionObserver : PgClientFlowObserver +{ + internal static readonly AdoCommandExecutionObserver Instance = new(); + protected internal override void OnCompleting( + PgClientFlow flow, Exception? exception, object? state) + => ((AdoCommandExecutionFlow)flow).CompleteLifetime(exception); } static class AdoCommandResultObserver @@ -123,28 +107,29 @@ internal static void DispatchIndexed(CommandResult result, object? state) } } -sealed class AdoCommandFlow : CommandFlow - where TCommand : IAdoCommand +sealed class AdoCommandExecutionFlow : PgClientFlow, IValueTaskSource, IValueTaskSource { - readonly FieldRef> _core; + readonly IAdoCommandExecutionOwner _bindingOwner; readonly DbParameterCollection? _parameters; readonly CommandBehavior _behavior; readonly SlonDataSource.PgDbDependencies _dependencies; readonly SlonConnection? _connection; readonly bool _preparing; readonly int _commandCount; - object? _lifetimeOwner; + IAdoCommandExecutionOwner? _lifetimeOwner; Action? _resultObserver; object? _resultObserverState; + CommandExecutionState _state; - internal AdoCommandFlow( - bool async, FieldRef> core, + internal AdoCommandExecutionFlow( + bool async, IAdoCommandExecutionOwner bindingOwner, DbParameterCollection? parameters, CommandBehavior behavior, SlonDataSource.PgDbDependencies dependencies, SlonConnection? connection, - TimeSpan? pendingTimeout, bool preparing, int commandCount, object? lifetimeOwner) - : base(async, pendingTimeout) + TimeSpan? pendingTimeout, bool preparing, int commandCount, + IAdoCommandExecutionOwner? lifetimeOwner) + : base(supportsDeferredFlush: true) { - _core = core; + _bindingOwner = bindingOwner; _parameters = parameters; _behavior = behavior; _dependencies = dependencies; @@ -152,36 +137,126 @@ internal AdoCommandFlow( _preparing = preparing; _commandCount = commandCount; _lifetimeOwner = lifetimeOwner; - SetObserver(AdoCommandFlowObserver.Instance, null); - AdoCommandFlowObserver.Instance.OnStarted(this, null); + _state.CommandIndex = -1; + _state.EnableActivationTimeout = true; + _state.WaitForDrainOnDispose = true; + _state.PendingTimeout = pendingTimeout; + IsAsync = async; + if (!async) + _state.HandoffEvent = new(false); + SetObserver(AdoCommandExecutionObserver.Instance, null); + lifetimeOwner?.OnFlowStarted(this); } - internal override int VisibleCommandCount => _commandCount; - internal object? LifetimeOwner => _lifetimeOwner; + internal override bool DefersSyncHandoff => true; + private protected override FlowHandoffEvent? HandoffEvent => _state.HandoffEvent; + protected override bool EnableActivationTimeout => true; + protected override TimeSpan? PendingTimeout => _state.PendingTimeout; + internal override TimeSpan? BackendCancellationGracePeriod + => Volatile.Read(ref _state.ConsumerDetached) ? TimeSpan.FromSeconds(1) : null; + + internal override void BindCallerToken(CancellationToken cancellationToken) + => _state.FlowToken = cancellationToken; + internal override CancellationToken MigrationCancellationToken => _state.FlowToken; - internal void ObserveResult(CommandResult result) + internal int VisibleCommandCount => _commandCount; + internal CommandResult? CurrentResult => _state.Current; + internal bool IsResultReady => Core.IsResultReady; + internal bool HasCancellationState => _state.ColdState is not null; + + public Enumerator GetEnumerator() => new(this, default); + + public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + if (cancellationToken.CanBeCanceled) + _state.FlowToken = cancellationToken; + return new(this, cancellationToken); + } + + CommandExecutionCore Core => new(new(this)); + + internal ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) + => Core.ConsumeNonQueryAsync(cancellationToken); + + protected override ValueTask ExecuteAuto(Context context) => Core.ExecuteAuto(context); + internal Task CancelAsync() => Core.CancelAsync(); + internal override bool ResetsSharedReadStateBeforeRelease => true; + protected override void OnStopping(Exception exception) => Core.OnStopping(exception); + protected override void OnAbort(Exception exception) => Core.OnAbort(exception); + internal override void Fail(Exception exception) => Core.Fail(exception); + protected override void OnReleasing(Exception? exception) => Core.OnReleasing(exception); + protected override void OnDiscarded() => Core.OnDiscarded(); + protected override void OnReset() => Core.OnReset(); + + void ObserveResult(CommandResult result) => _resultObserver?.Invoke(result, _resultObserverState); + internal void CompleteLifetime(Exception? exception) + => Interlocked.Exchange(ref _lifetimeOwner, null)?.OnFlowCompleting(this, exception); + internal override void Bind(PgClientFlowBindingContext? context) { var pgConnection = (context as PgConnection.FlowBindingContext)?.Connection ?? throw new InvalidOperationException( "An ADO command requires a PgConnection flow binding context."); - ref var core = ref _core.Invoke(); - InitializeAdo(IsAsync, core.CreateAdoCommandFlowOptions( - [_parameters], _behavior, _dependencies, _connection, pgConnection, - pendingTimeout: PendingTimeout, preparing: _preparing)); + var options = _bindingOwner.CreateExecutionOptions( + _parameters, _behavior, _dependencies, _connection, pgConnection, + PendingTimeout, _preparing); + _resultObserver = options.ResultObserver; + _resultObserverState = options.ResultObserverState; + _state.Commands = options.Commands; + _state.PendingTimeout = options.PendingTimeout; } - void InitializeAdo(bool async, in AdoCommandFlowOptions options) + readonly struct Ops(AdoCommandExecutionFlow owner) : ICommandExecutionFlowOps { - _resultObserver = options.ResultObserver; - _resultObserverState = options.ResultObserverState; - Initialize(async, new CommandFlowOptions + readonly AdoCommandExecutionFlow _owner = owner; + + public static Ops Create(PgClientFlow flow) => new((AdoCommandExecutionFlow)flow); + public PgClientFlow Flow => _owner; + public ref CommandExecutionState State => ref _owner._state; + public bool IsAsync { - Commands = options.Commands, - PendingTimeout = options.PendingTimeout - }); + get => _owner.IsAsync; + set => _owner.IsAsync = value; + } + public bool IsAsyncAtDispatch => _owner.IsAsyncAtDispatch; + public bool HasSuccessfulActivation => _owner.HasSuccessfulActivation; + public void WaitForSyncHandoff() => _owner.WaitForSyncHandoff(); + public void OnCommandResult(CommandResult result) => _owner.ObserveResult(result); + public void OnDrainStarted() { } + public void OnDiscarded() => _owner.CompleteLifetime(null); + } + + bool IValueTaskSource.GetResult(short token) => _state.ReadySource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) + => _state.ReadySource.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, + ValueTaskSourceOnCompletedFlags flags) + => _state.ReadySource.OnCompleted(continuation, state, token, flags); + + void IValueTaskSource.GetResult(short token) => _state.PipelineTaskSource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) + => _state.PipelineTaskSource.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, + ValueTaskSourceOnCompletedFlags flags) + => _state.PipelineTaskSource.OnCompleted(continuation, state, token, flags); + + public readonly struct Enumerator : IAsyncEnumerator, IDisposable + { + readonly AdoCommandExecutionFlow? _flow; + readonly CancellationToken _cancellationToken; + + internal Enumerator(AdoCommandExecutionFlow flow, CancellationToken cancellationToken) + => (_flow, _cancellationToken) = (flow, cancellationToken); + + public bool MoveNext() => _flow?.Core.MoveNext() ?? false; + public ValueTask MoveNextAsync() => MoveNextAsync(_cancellationToken); + public ValueTask MoveNextAsync(CancellationToken cancellationToken) + => _flow is null ? new(false) : _flow.Core.MoveNextAsync(cancellationToken); + public CommandResult Current => _flow?._state.Current ?? default!; + public ValueTask DisposeAsync() => _flow is null ? default : _flow.Core.DisposeAsync(); + public void Dispose() => _flow?.Core.Dispose(); } } diff --git a/Slon/Ado/AdoConnectionProxy.cs b/Slon/Ado/AdoConnectionProxy.cs index 36bc790..2b39c18 100644 --- a/Slon/Ado/AdoConnectionProxy.cs +++ b/Slon/Ado/AdoConnectionProxy.cs @@ -48,7 +48,7 @@ internal ConnectionState State return ConnectionState.Open; var activatedFlow = scope.ActivatedFlow; - if (activatedFlow is CommandFlow { IsResultReady: true }) + if (activatedFlow is AdoCommandExecutionFlow { IsResultReady: true }) return ConnectionState.Fetching; return activatedFlow is not null || scope.ExecutingFlow is not null diff --git a/Slon/SlonBatch.cs b/Slon/SlonBatch.cs index 5766bdc..54bee1c 100644 --- a/Slon/SlonBatch.cs +++ b/Slon/SlonBatch.cs @@ -32,17 +32,32 @@ unsafe SlonBatch(SlonConnection? connection, SlonDataSource? dataSource) unsafe SlonBatchCommands CreateBatchCommandCollection() => new(FieldRef>.Create(&GetBatchCore, this)); - internal void OnFlowStarted(CommandFlow flow) + internal void OnFlowStarted(AdoCommandExecutionFlow flow) => _batchCore.OnFlowStarted(flow); - internal void OnFlowCompleting(CommandFlow flow, Exception? exception) + internal void OnFlowCompleting(AdoCommandExecutionFlow flow, Exception? exception) => _batchCore.OnFlowCompleting(flow, exception); + AdoCommandFlowOptions IAdoCommandExecutionOwner.CreateExecutionOptions( + DbParameterCollection? parameters, CommandBehavior behavior, + SlonDataSource.PgDbDependencies dependencies, SlonConnection? connection, + PgConnection pgConnection, TimeSpan? pendingTimeout, bool preparing) + => _batchCore.CreateAdoCommandFlowOptions( + [parameters], behavior, dependencies, connection, pgConnection, + pendingTimeout, preparing); + + void IAdoCommandExecutionOwner.OnFlowStarted(AdoCommandExecutionFlow flow) + => OnFlowStarted(flow); + + void IAdoCommandExecutionOwner.OnFlowCompleting( + AdoCommandExecutionFlow flow, Exception? exception) + => OnFlowCompleting(flow, exception); + static ref AdoBatchCore GetBatchCore(SlonBatch instance) => ref instance._batchCore; } // Public surface & ADO.NET -public sealed partial class SlonBatch : DbBatch +public sealed partial class SlonBatch : DbBatch, IAdoCommandExecutionOwner { /// Initializes an unbound batch. public SlonBatch() : this(null, null) {} diff --git a/Slon/SlonCommand.cs b/Slon/SlonCommand.cs index 5226525..1986a95 100644 --- a/Slon/SlonCommand.cs +++ b/Slon/SlonCommand.cs @@ -66,12 +66,27 @@ void Rebuild() } } - internal void OnFlowStarted(CommandFlow flow) + internal void OnFlowStarted(AdoCommandExecutionFlow flow) => _batchCore.OnFlowStarted(flow); - internal void OnFlowCompleting(CommandFlow flow, Exception? exception) + internal void OnFlowCompleting(AdoCommandExecutionFlow flow, Exception? exception) => _batchCore.OnFlowCompleting(flow, exception); + AdoCommandFlowOptions IAdoCommandExecutionOwner.CreateExecutionOptions( + DbParameterCollection? parameters, CommandBehavior behavior, + SlonDataSource.PgDbDependencies dependencies, SlonConnection? connection, + PgConnection pgConnection, TimeSpan? pendingTimeout, bool preparing) + => _batchCore.CreateAdoCommandFlowOptions( + [parameters], behavior, dependencies, connection, pgConnection, + pendingTimeout, preparing); + + void IAdoCommandExecutionOwner.OnFlowStarted(AdoCommandExecutionFlow flow) + => OnFlowStarted(flow); + + void IAdoCommandExecutionOwner.OnFlowCompleting( + AdoCommandExecutionFlow flow, Exception? exception) + => OnFlowCompleting(flow, exception); + struct AdoCommand : IAdoCommand { public void MakeReadOnly() { } @@ -85,7 +100,7 @@ public void MakeReadOnly() { } } // Public surface & ADO.NET -public sealed partial class SlonCommand : DbCommand +public sealed partial class SlonCommand : DbCommand, IAdoCommandExecutionOwner { /// Initializes an unbound command. public SlonCommand() : this(null, null, null) {} diff --git a/Slon/SlonDataReader.cs b/Slon/SlonDataReader.cs index 67cffd7..71847a1 100644 --- a/Slon/SlonDataReader.cs +++ b/Slon/SlonDataReader.cs @@ -39,12 +39,12 @@ ReaderState State CommandResult.RowEnumerator _rowEnumerator; PgSerializerFieldReader _fieldReader; int _remainingResults; - CommandFlow.Enumerator _enumerator; + AdoCommandExecutionFlow.Enumerator _enumerator; long? _recordsAffected; SlonDataReader() { } - void Initialize(CommandFlow.Enumerator enumerator, CommandBehavior behavior, int remainingResults, + void Initialize(AdoCommandExecutionFlow.Enumerator enumerator, CommandBehavior behavior, int remainingResults, PgSerializerOptions serializerOptions, SlonConnection? connectionToClose, long? recordsAffected, bool hasCurrent) { @@ -84,7 +84,7 @@ static int GetResultLimit(CommandBehavior behavior, int commandCount) static bool ShouldEnumerateCommandResults(CommandBehavior behavior) => behavior.HasFlag(EnumerateCommandResultsBehavior); - static SlonDataReader CreateReader(CommandFlow.Enumerator enumerator, CommandBehavior behavior, + static SlonDataReader CreateReader(AdoCommandExecutionFlow.Enumerator enumerator, CommandBehavior behavior, int remainingResults, PgSerializerOptions serializerOptions, SlonConnection? connectionToClose, long? recordsAffected, bool hasCurrent) { @@ -99,7 +99,7 @@ static SlonDataReader CreateReader(CommandFlow.Enumerator enumerator, CommandBeh CommandResult? Current => _enumerator.Current; bool IsSequential => _rowBuffering is CommandResult.RowBuffering.Streaming; - internal static SlonDataReader Create(CommandBehavior behavior, CommandFlow flow, + internal static SlonDataReader Create(CommandBehavior behavior, AdoCommandExecutionFlow flow, PgSerializerOptions serializerOptions, SlonConnection? connectionToClose = null) { @@ -127,13 +127,13 @@ internal static SlonDataReader Create(CommandBehavior behavior, CommandFlow flow } internal static async ValueTask CreateAsync(CommandBehavior behavior, - ValueTask flowTask, PgSerializerOptions serializerOptions, + ValueTask flowTask, PgSerializerOptions serializerOptions, CancellationToken cancellationToken = default, SlonConnection? connectionToClose = null, Activity? activity = null) where TReader : DbDataReader { Debug.Assert(typeof(TReader) == typeof(SlonDataReader) || typeof(TReader) == typeof(DbDataReader)); - CommandFlow.Enumerator enumerator = default; + AdoCommandExecutionFlow.Enumerator enumerator = default; try { var flow = await flowTask.ConfigureAwait(false); @@ -473,7 +473,7 @@ async ValueTask DisposeEnumeratorAsync() } } - (CommandResult.RowEnumerator Rows, CommandFlow.Enumerator Results) BeginEnumeratorDisposal() + (CommandResult.RowEnumerator Rows, AdoCommandExecutionFlow.Enumerator Results) BeginEnumeratorDisposal() { if (_enumeratorDisposalActive) ThrowHelper.ThrowInvalidOperation("Invalid concurrent call."); diff --git a/Slon/SlonDataSource.cs b/Slon/SlonDataSource.cs index 3e77245..f6d94ff 100644 --- a/Slon/SlonDataSource.cs +++ b/Slon/SlonDataSource.cs @@ -81,7 +81,7 @@ internal void ReportTransactionDisposeRollbackFailure(Exception exception) // The multiplexed path lets the pool select a wire before materializing connection-local command // state. A rejected candidate rolls that attempt back and reuses the still-unqueued flow shell. - internal CommandFlow EnqueueCommands(CommandFlow flow, TimeSpan pendingTimeout) + internal AdoCommandExecutionFlow EnqueueCommands(AdoCommandExecutionFlow flow, TimeSpan pendingTimeout) { try { @@ -95,8 +95,8 @@ internal CommandFlow EnqueueCommands(CommandFlow flow, TimeSpan pendingTimeout) } } - internal async ValueTask EnqueueCommandsAsync( - CommandFlow flow, TimeSpan pendingTimeout, CancellationToken cancellationToken) + internal async ValueTask EnqueueCommandsAsync( + AdoCommandExecutionFlow flow, TimeSpan pendingTimeout, CancellationToken cancellationToken) { try { @@ -111,7 +111,7 @@ await _connectionPool.GetAsync(static (ctx, f) => TrySchedule(ctx, f), flow, pen } } - static bool TrySchedule(ConnectionCandidate context, CommandFlow flow) + static bool TrySchedule(ConnectionCandidate context, AdoCommandExecutionFlow flow) { var enqueueOptions = context.IsIdleCandidate ? FlowEnqueueOptions.AllowMigration From 982cb2341d51d536ef9caf0ac61f7794a13ccd0f Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 01:42:30 +0200 Subject: [PATCH 089/136] Make the replacement the public command flow --- Slon.Tests/Ado/FlowMigrationTests.cs | 7 + Slon.Tests/CommandFlowImplementation.cs | 9 +- Slon.Tests/FlowBindingProbe.cs | 2 +- .../Pg/PublicCommandFlowSurfaceTests.cs | 29 + .../Pg/Protocol/Flows/CommandExecutionFlow.cs | 1520 ----------- .../Flows/CommandFlow.MessageEnumerator.cs | 2 +- Slon/Pg/Protocol/Flows/CommandFlow.cs | 2281 +++++++++-------- ...tor.cs => LegacyCommandFlow.Enumerator.cs} | 12 +- Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs | 1331 ++++++++++ Slon/Pg/Protocol/PgClientFlow.cs | 2 +- .../Protocol/PgClientProtocol.Cancellation.cs | 2 +- 11 files changed, 2621 insertions(+), 2576 deletions(-) create mode 100644 Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs delete mode 100644 Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs rename Slon/Pg/Protocol/Flows/{CommandFlow.Enumerator.cs => LegacyCommandFlow.Enumerator.cs} (98%) create mode 100644 Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs diff --git a/Slon.Tests/Ado/FlowMigrationTests.cs b/Slon.Tests/Ado/FlowMigrationTests.cs index 320d5df..0f6da67 100644 --- a/Slon.Tests/Ado/FlowMigrationTests.cs +++ b/Slon.Tests/Ado/FlowMigrationTests.cs @@ -183,4 +183,11 @@ static async Task DrainAsync(CommandFlow flow) await e.DisposeAsync(); } + static async Task DrainAsync(BindingProbeFlow flow) + { + var e = flow.GetAsyncEnumerator(); + while (await e.MoveNextAsync()) { } + await e.DisposeAsync(); + } + } diff --git a/Slon.Tests/CommandFlowImplementation.cs b/Slon.Tests/CommandFlowImplementation.cs index b71bc62..ac92053 100644 --- a/Slon.Tests/CommandFlowImplementation.cs +++ b/Slon.Tests/CommandFlowImplementation.cs @@ -1,5 +1,6 @@ -#if COMMAND_FLOW_NEXT -global using CommandFlow = Slon.Pg.Protocol.Flows.CommandExecutionFlow; -global using CommandFlowObserver = Slon.Pg.Protocol.Flows.CommandExecutionFlowObserver; -global using CommandFlowOptions = Slon.Pg.Protocol.Flows.CommandExecutionFlowOptions; +#if !COMMAND_FLOW_NEXT +global using CommandFlow = Slon.Pg.Protocol.Flows.LegacyCommandFlow; +global using CommandFlowObserver = Slon.Pg.Protocol.Flows.LegacyCommandFlowObserver; +global using CommandFlowOptions = Slon.Pg.Protocol.Flows.LegacyCommandFlowOptions; #endif +global using ReplacementCommandFlow = Slon.Pg.Protocol.Flows.CommandFlow; diff --git a/Slon.Tests/FlowBindingProbe.cs b/Slon.Tests/FlowBindingProbe.cs index b011a6d..e655860 100644 --- a/Slon.Tests/FlowBindingProbe.cs +++ b/Slon.Tests/FlowBindingProbe.cs @@ -1,7 +1,7 @@ using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; using Slon.Pg; -using LegacyCommandFlow = Slon.Pg.Protocol.Flows.CommandFlow; +using LegacyCommandFlow = Slon.Pg.Protocol.Flows.LegacyCommandFlow; namespace Slon.Tests; diff --git a/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs b/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs new file mode 100644 index 0000000..f878597 --- /dev/null +++ b/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs @@ -0,0 +1,29 @@ +using System.Collections; +using Slon.Pg; +using Slon.Pg.Protocol.Flows; + +namespace Slon.Tests.Pg; + +[TestClass] +public class PublicCommandFlowSurfaceTests +{ + [TestMethod] + public void ReplacementIsTheSealedPublicCommandFlowSurface() + { + var flow = typeof(Slon.Pg.Protocol.Flows.CommandFlow); + Assert.IsTrue(flow.IsPublic); + Assert.IsTrue(flow.IsSealed); + Assert.IsTrue(typeof(Slon.Pg.Protocol.Flows.CommandFlowObserver).IsPublic); + Assert.IsTrue(typeof(Slon.Pg.Protocol.Flows.CommandFlowOptions).IsPublic); + Assert.IsTrue(typeof(IEnumerator) + .IsAssignableFrom(typeof(Slon.Pg.Protocol.Flows.CommandFlow.Enumerator))); + + Assert.IsNotNull(flow.GetConstructor([ + typeof(bool), typeof(ReadOnlySpan) + ])); + Assert.IsNotNull(flow.GetConstructor([ + typeof(bool), typeof(Slon.Pg.Protocol.Flows.CommandFlowOptions).MakeByRefType() + ])); + Assert.IsFalse(typeof(LegacyCommandFlow).IsPublic); + } +} diff --git a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs b/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs deleted file mode 100644 index 1c05887..0000000 --- a/Slon/Pg/Protocol/Flows/CommandExecutionFlow.cs +++ /dev/null @@ -1,1520 +0,0 @@ -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; -using System.Runtime.InteropServices; -using System.Threading.Tasks.Sources; -using Slon.Runtime.CompilerServices; - -namespace Slon.Pg.Protocol.Flows; - -internal abstract class CommandExecutionFlowObserver : PgClientFlowObserver -{ - protected internal virtual void OnStarted(CommandExecutionFlow flow, object? state) { } - protected internal virtual void OnCommandResult( - CommandExecutionFlow flow, CommandResult result, object? state) { } - protected internal virtual void OnDrainStarted(CommandExecutionFlow flow, object? state) { } -} - -internal readonly struct CommandExecutionFlowOptions -{ - public CommandExecutionFlowObserver? Observer { get; init; } - public object? ObserverState { get; init; } - public CommandList Commands { get; init; } - public TimeSpan? PendingTimeout { get; init; } -} - -internal sealed class CommandExecutionColdState -{ - internal bool CancelRequested; - internal CancellationToken CallerToken; - internal CancellationTokenRegistration CallerRegistration; - internal CancellationToken DeliverToken; - internal int Scope; - internal int Timing; - internal int SubsequentTiming; - internal TaskCompletionSource? Delivery; - internal object? EpisodeKey; - internal Exception? CloseException; - // Replayed by later consumer calls once the flow reached its terminal. - internal Exception? TerminalException; - // Command errors observed while draining without a consumer. Multiple Sync windows may each - // produce an independent ErrorResponse, all of which belong to the waiting disposer. - internal List? DrainErrors; -} - -internal enum CommandExecutionCancellationScope : byte -{ - CurrentWindow = 1, - RemainingFlow = 2 -} - -// Mutable execution state is stored inline by each concrete host. Core algorithms re-enter this -// field through their host-specific ops value after every await; they never mutate a copied struct. -[StructLayout(LayoutKind.Auto)] -internal struct CommandExecutionState -{ - internal int Phase; - internal CommandList Commands; - internal TimeSpan? PendingTimeout; - internal int CommandIndex; - internal PgClientFlow.Context Context; - internal bool ContextPublished; - internal CommandResult? Current; - internal bool CurrentPublished; - internal bool ReadFlowRfq; - internal bool ConsumerDetached; - internal bool ConsumerObservedCompletion; - internal Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore ReadySource; - internal int ReadyCompletion; - internal Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore PipelineTaskSource; - internal CancellationToken FlowToken; - internal CancellationTokenRegistration FlowRegistration; - internal CommandExecutionColdState? ColdState; - internal FlowHandoffEvent? HandoffEvent; - internal bool SyncHandoffClaimed; - internal int DrainStarted; - internal bool EnableActivationTimeout; - internal bool WaitForDrainOnDispose; -} - -// Replacement-flow prototype: one consumer-owned decoder lifecycle for synchronous and asynchronous -// execution, with the general multi-command result shape. Kept internal until it covers the complete -// CommandFlow contract; the eventual single-command specialization will remain a separate sealed type. -internal sealed partial class CommandExecutionFlow : PgClientFlow, IValueTaskSource, IValueTaskSource -{ - static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); - CommandExecutionState _state; - CommandExecutionFlowObserver? _commandObserver; - object? _commandObserverState; - - CommandExecutionFlow(bool async, TimeSpan? pendingTimeout = null) - : base(supportsDeferredFlush: true) - { - _state.CommandIndex = -1; - _state.EnableActivationTimeout = true; - _state.WaitForDrainOnDispose = true; - _state.PendingTimeout = pendingTimeout; - IsAsync = async; - if (!async) - _state.HandoffEvent = new(false); - } - - internal CommandExecutionFlow(bool async, params ReadOnlySpan commands) - : this(async) - => Initialize(async, commands); - - internal CommandExecutionFlow( - bool async, bool enableActivationTimeout, params ReadOnlySpan commands) - : this(async, commands) - => _state.EnableActivationTimeout = enableActivationTimeout; - - internal CommandExecutionFlow(bool async, CommandList commands, TimeSpan? pendingTimeout = null) - : this(async, pendingTimeout) - => Initialize(async, new CommandExecutionFlowOptions - { - Commands = commands, - PendingTimeout = pendingTimeout - }); - - internal CommandExecutionFlow(bool async, in CommandExecutionFlowOptions options) - : this(async, options.PendingTimeout) - => Initialize(async, options); - - internal CommandExecutionFlow Initialize(bool async, params ReadOnlySpan commands) - => Initialize(async, new CommandExecutionFlowOptions { Commands = new(commands) }); - - internal CommandExecutionFlow Initialize(bool async, in CommandExecutionFlowOptions options) - { - IsAsync = async; - if (!async) - _state.HandoffEvent ??= new(false); - var commands = options.Commands; - if (commands.Count is 0) - return this; - _state.Commands = commands; - _state.PendingTimeout = options.PendingTimeout; - _commandObserver = options.Observer; - _commandObserverState = options.ObserverState; - if (options.Observer is { } observer) - { - SetObserver(observer, options.ObserverState); - observer.OnStarted(this, options.ObserverState); - } - return this; - } - - internal override bool DefersSyncHandoff => true; - private protected override FlowHandoffEvent? HandoffEvent => _state.HandoffEvent; - protected override bool EnableActivationTimeout => _state.EnableActivationTimeout; - protected override TimeSpan? PendingTimeout => _state.PendingTimeout; - internal override TimeSpan? BackendCancellationGracePeriod - => Volatile.Read(ref _state.ConsumerDetached) - ? ConsumerDrainCancellationGracePeriod - : null; - - internal override void BindCallerToken(CancellationToken cancellationToken) - => _state.FlowToken = cancellationToken; - internal override CancellationToken MigrationCancellationToken - => _state.FlowToken; - - public Enumerator GetEnumerator() - => new(this, default); - - public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) - { - // A missing enumeration token must not erase the token captured when the flow was queued. - if (cancellationToken.CanBeCanceled) - _state.FlowToken = cancellationToken; - return new(this, cancellationToken); - } - - internal CommandResult? CurrentResult => _state.Current; - internal bool IsResultReady => Core.IsResultReady; - internal int VisibleCommandCount => _state.Commands.Count; - internal ValueTask MoveNextResultAsync(CancellationToken cancellationToken) - => Core.MoveNextAsync(cancellationToken); - internal void DisposeResults() => Core.Dispose(); - internal ValueTask DisposeResultsAsync() => Core.DisposeAsync(); - internal bool WaitForDrainOnDispose - { - get => _state.WaitForDrainOnDispose; - set => _state.WaitForDrainOnDispose = value; - } - - CommandExecutionCore Core => new(new(this)); - - internal ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) - => Core.ConsumeNonQueryAsync(cancellationToken); - - protected override ValueTask ExecuteAuto(Context context) - => Core.ExecuteAuto(context); - - internal Task CancelAsync() => Core.CancelAsync(); - - internal override bool ResetsSharedReadStateBeforeRelease => true; - protected override void OnStopping(Exception exception) => Core.OnStopping(exception); - protected override void OnAbort(Exception exception) => Core.OnAbort(exception); - internal override void Fail(Exception exception) => Core.Fail(exception); - protected override void OnReleasing(Exception? exception) => Core.OnReleasing(exception); - protected override void OnDiscarded() => Core.OnDiscarded(); - protected override void OnReset() => Core.OnReset(); - - readonly struct Ops(CommandExecutionFlow owner) : ICommandExecutionFlowOps - { - readonly CommandExecutionFlow _owner = owner; - - public static Ops Create(PgClientFlow flow) => new((CommandExecutionFlow)flow); - public PgClientFlow Flow => _owner; - public ref CommandExecutionState State => ref _owner._state; - public bool IsAsync - { - get => _owner.IsAsync; - set => _owner.IsAsync = value; - } - public bool IsAsyncAtDispatch => _owner.IsAsyncAtDispatch; - public bool HasSuccessfulActivation => _owner.HasSuccessfulActivation; - public void WaitForSyncHandoff() => _owner.WaitForSyncHandoff(); - public void OnCommandResult(CommandResult result) - => _owner._commandObserver?.OnCommandResult( - _owner, result, _owner._commandObserverState); - public void OnDrainStarted() - => _owner._commandObserver?.OnDrainStarted( - _owner, _owner._commandObserverState); - public void OnDiscarded() - => _owner.GetObserver(out var observerState)?.OnCompleting( - _owner, null, observerState); - } -} - -internal interface ICommandExecutionFlowOps - where TSelf : struct, ICommandExecutionFlowOps -{ - static abstract TSelf Create(PgClientFlow flow); - PgClientFlow Flow { get; } - ref CommandExecutionState State { get; } - bool IsAsync { get; set; } - bool IsAsyncAtDispatch { get; } - bool HasSuccessfulActivation { get; } - void WaitForSyncHandoff(); - void OnCommandResult(CommandResult result); - void OnDrainStarted(); - void OnDiscarded(); -} - -readonly struct CommandExecutionCore(TOps ops) - where TOps : struct, ICommandExecutionFlowOps -{ - const int PhaseInitial = 0; - const int PhaseReading = 1; - const int PhaseResultReady = 2; - const int PhaseDraining = 3; - const int PhaseCompleted = 4; - - readonly TOps _ops = ops; - ref CommandExecutionState _state => ref _ops.State; - internal bool IsResultReady => Volatile.Read(ref _state.Phase) is PhaseResultReady; - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - internal async ValueTask ConsumeNonQueryAsync( - CancellationToken cancellationToken = default) - { - var recordsAffected = -1L; - if (cancellationToken.CanBeCanceled) - _state.FlowToken = cancellationToken; - try - { - while (await MoveNextAsync(cancellationToken).ConfigureAwait(false)) - { - var result = _state.Current!; - await result.CompleteAsync().ConfigureAwait(false); - var affected = result.GetCommandComplete().BatchRecordsAffected; - if (affected >= 0) - recordsAffected = recordsAffected < 0 - ? affected - : checked(recordsAffected + affected); - } - return recordsAffected; - } - finally - { - await DisposeAsync().ConfigureAwait(false); - } - } - - CommandExecutionColdState GetOrCreateColdState() - => Volatile.Read(ref _state.ColdState) ?? - Interlocked.CompareExchange(ref _state.ColdState, new(), null) ?? _state.ColdState; - - bool IsClosed => Volatile.Read(ref _state.ColdState)?.CloseException is not null; - bool IsCancelRequested => Volatile.Read(ref _state.ColdState) is { CancelRequested: true }; - bool HasDecoder => _state.ContextPublished && _ops.HasSuccessfulActivation; - - internal ValueTask ExecuteAuto(PgClientFlow.Context context) - { - _state.Context = context; - _state.ContextPublished = true; - ValueTask writeTask; - try - { - ref readonly var template = ref _state.Commands.ItemRef(_state.Commands.Count - 1); - var appendSync = !template.WithSync; - _state.ReadFlowRfq = appendSync; - // Caller cancellation never cancels wire I/O. The consumer observes the latched intent and - // drains its command to RFQ instead. - writeTask = _ops.IsAsync - ? _state.Commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) - : WriteCommandsResumable(context, appendSync); - // Observe synchronous faults here; pending writes remain the framework-owned trailing task. - if (writeTask.IsCompleted) - writeTask.GetAwaiter().GetResult(); - } - catch (Exception ex) - { - // The framework recovers the wire from this throw. Only the consumer needs the fault. - FaultReady(ex); - throw; - } - - // Activation may precede or follow execution. Bridging it here guarantees a consumer resumes - // against a published context, and delivers an activation fault to the pipeline task when no - // consumer ever arrives. - var activation = context.GetDecoderAsync().ConfigureAwait(false); - if (activation.IsCompleted) - OnActivationSettled(onExecutorStrand: true); - else - activation.UnsafeOnCompleted(static state => - new CommandExecutionCore(TOps.Create((PgClientFlow)state!)) - .OnActivationSettled(onExecutorStrand: false), _ops.Flow); - return new(new FlowTasks(writeTask, new ValueTask((IValueTaskSource)_ops.Flow, _state.PipelineTaskSource.Version))); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - ValueTask WriteCommandsResumable(PgClientFlow.Context context, bool appendSync) - { - var encoder = context.GetEncoder(); - ValueTask writeTask; - using (encoder.BeginResumableWriteScope()) - writeTask = _state.Commands.WriteCommandsResumable(encoder, appendSync); - return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); - } - - // Runs on the executor strand when activation already settled, else on the activation dispatch. - // The executor strand never runs consumer code. An activation dispatch is a detached work item - // whose only remaining work is this wake, so the consumer may continue on it directly. - void OnActivationSettled(bool onExecutorStrand) - { - try - { - _ = _state.Context.GetDecoderAsync().ConfigureAwait(false).GetAwaiter().GetResult(); - } - catch (Exception fault) - { - // Preserve the activation's close, timeout, or cancellation identity. The pipeline task - // faults first so a consumer woken by the ready source never races its retirement. - CompletePipelineTask(fault, runContinuationsAsynchronously: true); - FaultReady(fault); - return; - } - - if (IsCancelRequested) - RequestBackendCancellation(); - if (!CompleteReady(null, runContinuationsAsynchronously: onExecutorStrand)) - { - // Teardown released the consumer while this flow waited for its turn. Nothing reads the - // response, the closing wire owns it. - CompletePipelineTask(null, runContinuationsAsynchronously: true); - return; - } - // A cancel latched before activation may have released its caller already. The response - // still has to reach RFQ, so drain it unless a consumer already owns the decoder. - if (IsCancelRequested) - TryTakeOverDrain(); - } - - void FaultReady(Exception exception) - { - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - CompleteReady(exception, runContinuationsAsynchronously: true); - } - - bool CompleteReady(Exception? exception, bool runContinuationsAsynchronously) - { - if (Interlocked.CompareExchange(ref _state.ReadyCompletion, 1, 0) != 0) - return false; - if (exception is null) - _state.ReadySource.SetResult(true, runContinuationsAsynchronously); - else - _state.ReadySource.SetException(exception, runContinuationsAsynchronously); - return true; - } - - void CompletePipelineTask(Exception? exception, bool runContinuationsAsynchronously = false) - { - if (Interlocked.Exchange(ref _state.Phase, PhaseCompleted) is PhaseCompleted) - return; - if (exception is null) - _state.PipelineTaskSource.SetResult(true, runContinuationsAsynchronously); - else - _state.PipelineTaskSource.SetException(exception, runContinuationsAsynchronously); - } - - void EnsureSyncHandoff() - { - if (_ops.IsAsyncAtDispatch) - ThrowHelper.ThrowInvalidOperation( - "Synchronous result consumption requires a flow initialized for synchronous execution."); - if (_state.SyncHandoffClaimed) - return; - _ops.WaitForSyncHandoff(); - _state.SyncHandoffClaimed = true; - } - - internal bool MoveNext() - { - EnsureSyncHandoff(); - while (true) - { - var phase = Volatile.Read(ref _state.Phase); - switch (phase) - { - case PhaseInitial: - if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseInitial) != PhaseInitial) - continue; - _state.CommandIndex = 0; - return First(); - case PhaseResultReady: - if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) != PhaseResultReady) - continue; - return NextBatch(); - case PhaseReading: - ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow."); - return false; - case PhaseDraining: - _ops.Flow.WaitForCompleteSynchronously(); - throw Volatile.Read(ref _state.ColdState)?.TerminalException - ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); - default: - if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } terminal) - ExceptionDispatchInfo.Throw(terminal); - return false; - } - } - } - - bool First() - { - try - { - WaitForReadySynchronously(); - Debug.Assert(!_state.ConsumerDetached); - RegisterCancellation(default); - var result = ReadNextPublishedResult(); - return result is not null && PublishSynchronousResult(result); - } - catch (TimeoutException ex) - { - HandleReadTimeout(ex); - throw; - } - catch (Exception ex) - { - FaultFromOwner(ex); - throw; - } - } - - bool NextBatch() - { - try - { - RegisterCancellation(default); - var result = _state.Current!; - var completeError = CompleteCurrentResult(); - _state.CurrentPublished = false; - if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } consumerFault) - { - Interlocked.Exchange(ref _state.Phase, PhaseDraining); - NotifyDrainStarted(); - _state.ConsumerDetached = true; - Drain(); - ExceptionDispatchInfo.Throw(consumerFault); - } - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) - SkipDiscardedCommands(); - - _state.CommandIndex++; - var next = ReadNextPublishedResult(); - if (next is not null) - return PublishSynchronousResult(next); - - return false; - } - catch (TimeoutException ex) - { - HandleReadTimeout(ex); - throw; - } - catch (Exception ex) - { - FaultFromOwner(ex); - throw; - } - } - - bool PublishSynchronousResult(CommandResult result) - { - _state.Current = result; - _state.CurrentPublished = true; - Interlocked.Exchange(ref _state.Phase, PhaseResultReady); - var context = _state.Context; - if (!IsClosed && context.StoppingToken.IsCancellationRequested) - Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, - context.FlowTerminationException, null); - if (!IsCancelRequested && !IsClosed) - return true; - - if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) - Drain(); - else - _ops.Flow.WaitForCompleteSynchronously(); - throw Volatile.Read(ref _state.ColdState)?.TerminalException - ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); - } - - void WaitForReadySynchronously() - { - var ready = new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version); - if (ready.IsCompleted) - _ = ready.GetAwaiter().GetResult(); - else - _ = ready.AsTask().GetAwaiter().GetResult(); - } - - internal ValueTask MoveNextAsync(CancellationToken cancellationToken) - { - if (!_ops.IsAsyncAtDispatch) - return ValueTask.FromException(ThrowHelper.ThrowInvalidOperation( - "Asynchronous result consumption requires a flow initialized for asynchronous execution.")); - while (true) - { - var phase = Volatile.Read(ref _state.Phase); - switch (phase) - { - case PhaseInitial: - if (cancellationToken.IsCancellationRequested) - return CancelBeforeRead(cancellationToken); - if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseInitial) != PhaseInitial) - continue; - _state.CommandIndex = 0; - return FirstAsync(cancellationToken); - case PhaseResultReady: - if (cancellationToken.IsCancellationRequested) - return CancelBeforeRead(cancellationToken); - if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) != PhaseResultReady) - continue; - return NextBatchAsync(cancellationToken); - case PhaseReading: - return ValueTask.FromException( - ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow.")); - case PhaseDraining: - return AwaitTakeoverAsync(); - default: - return Volatile.Read(ref _state.ColdState)?.TerminalException is { } terminal - ? ValueTask.FromException(terminal) - : new(false); - } - } - } - - // A pre-cancelled token releases the caller immediately. The wire still drains to RFQ. - ValueTask CancelBeforeRead(CancellationToken cancellationToken) - { - RequestCancel(cancellationToken, CommandExecutionCancellationScope.CurrentWindow); - return ValueTask.FromException(new OperationCanceledException(cancellationToken)); - } - - // The consumer parks behind a takeover drain and receives the outcome that caused it. - async ValueTask AwaitTakeoverAsync() - { - await WaitForCompletionAsync().ConfigureAwait(false); - throw Volatile.Read(ref _state.ColdState)?.TerminalException ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); - } - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask FirstAsync(CancellationToken cancellationToken) - { - Exception? deliver; - try - { - await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); - Debug.Assert(!_state.ConsumerDetached); - RegisterCancellation(cancellationToken); - var result = await ReadNextPublishedResultAsync().ConfigureAwait(false); - if (result is null) - return false; - _state.Current = result; - _state.CurrentPublished = true; - // Publish the idle state, then recheck the latches. A latch that landed between the read - // and this publication found no idle owner to take over, so this frame must act on it. - Interlocked.Exchange(ref _state.Phase, PhaseResultReady); - // Graceful stopping faults a result that arrives after the close began, as the ordinary - // flow does at each result boundary. Latch it so the drain delivers that close. - var context = _state.Context; - if (!IsClosed && context.StoppingToken.IsCancellationRequested) - Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, - context.FlowTerminationException, null); - if (!IsCancelRequested && !IsClosed) - return true; - if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) - { - await DrainAsync().ConfigureAwait(false); - } - else - { - // The latching side took the decoder first. Park behind its drain. - await WaitForCompletionAsync().ConfigureAwait(false); - } - deliver = Volatile.Read(ref _state.ColdState)?.TerminalException; - } - catch (TimeoutException ex) - { - HandleReadTimeout(ex); - throw; - } - catch (Exception ex) - { - FaultFromOwner(ex); - throw; - } - throw deliver ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); - } - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask NextBatchAsync(CancellationToken cancellationToken) - { - try - { - RegisterCancellation(cancellationToken); - var result = _state.Current!; - var resultEnumerator = _state.Context.GetProtocolStatic() - .ResultMessageEnumerator; - await resultEnumerator.DisposeAsync().ConfigureAwait(false); - var completeError = resultEnumerator.CompleteError; - _state.CurrentPublished = false; - if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } consumerFault) - { - Interlocked.Exchange(ref _state.Phase, PhaseDraining); - NotifyDrainStarted(); - _state.ConsumerDetached = true; - await DrainAsync().ConfigureAwait(false); - ExceptionDispatchInfo.Throw(consumerFault); - } - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) - await SkipDiscardedCommandsAsync().ConfigureAwait(false); - - _state.CommandIndex++; - var next = await ReadNextPublishedResultAsync().ConfigureAwait(false); - if (next is not null) - { - result = next; - _state.Current = result; - _state.CurrentPublished = true; - Interlocked.Exchange(ref _state.Phase, PhaseResultReady); - var context = _state.Context; - if (!IsClosed && context.StoppingToken.IsCancellationRequested) - Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, - context.FlowTerminationException, null); - if (!IsCancelRequested && !IsClosed) - return true; - - if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) - await DrainAsync().ConfigureAwait(false); - else - await WaitForCompletionAsync().ConfigureAwait(false); - throw Volatile.Read(ref _state.ColdState)?.TerminalException - ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); - } - - return false; - } - catch (TimeoutException ex) - { - HandleReadTimeout(ex); - throw; - } - catch (Exception ex) - { - FaultFromOwner(ex); - throw; - } - } - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask ReadNextPublishedResultAsync() - { - CommandResult? result = _state.Current; - while (_state.CommandIndex < _state.Commands.Count) - { - result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); - _state.Current = result; - _state.CurrentPublished = false; - if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) - return result; - - var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); - var suppressedError = result.Error; - if (suppressedError is null && completeError is { } completionError) - suppressedError = completionError.Error; - if (suppressedError is not null) - { - var exception = PgErrorException.Create(suppressedError); - var cold = GetOrCreateColdState(); - Interlocked.CompareExchange(ref cold.TerminalException, exception, null); - (cold.DrainErrors ??= new()).Add(exception); - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) - await SkipDiscardedCommandsAsync().ConfigureAwait(false); - _state.CommandIndex++; - _state.Current = null; - Interlocked.Exchange(ref _state.Phase, PhaseDraining); - NotifyDrainStarted(); - _state.ConsumerDetached = true; - await DrainAsync().ConfigureAwait(false); - throw exception; - } - - _state.CommandIndex++; - } - - if (result is null) - throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); - await CompleteBatchAsync().ConfigureAwait(false); - _state.ConsumerObservedCompletion = true; - return null; - } - - CommandResult? ReadNextPublishedResult() - { - CommandResult? result = _state.Current; - while (_state.CommandIndex < _state.Commands.Count) - { - result = ReadResult(_state.CommandIndex); - _state.Current = result; - _state.CurrentPublished = false; - if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) - return result; - - var completeError = CompleteCurrentResult(); - var suppressedError = result.Error; - if (suppressedError is null && completeError is { } completionError) - suppressedError = completionError.Error; - if (suppressedError is not null) - { - var exception = PgErrorException.Create(suppressedError); - var cold = GetOrCreateColdState(); - Interlocked.CompareExchange(ref cold.TerminalException, exception, null); - (cold.DrainErrors ??= new()).Add(exception); - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) - SkipDiscardedCommands(); - _state.CommandIndex++; - _state.Current = null; - Interlocked.Exchange(ref _state.Phase, PhaseDraining); - NotifyDrainStarted(); - _state.ConsumerDetached = true; - Drain(); - throw exception; - } - - _state.CommandIndex++; - } - - if (result is null) - throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); - CompleteBatch(); - _state.ConsumerObservedCompletion = true; - return null; - } - - // Reads through the command's execute prelude and initializes the protocol-static result. - async ValueTask ReadResultAsync(int commandIndex) - { - var context = _state.Context; - var decoder = context.Decoder; - // After close, a fresh command must not consume bytes left by its predecessor. - if (context.IsProtocolClosed) - throw context.FlowTerminationException; - PgError? error; - RowDescription? requestedRowDescription; - ref readonly var command = ref _state.Commands.ItemRef(commandIndex); - var describeOnly = command.DescribeOnly; - var hasPreparedDescription = command.Descriptor - is { IsPrepared: true, PreparedRowDescription: not null }; - decoder.UseReadTimeout(command.Timeout); - ParameterTypeList? preparationParameterTypes = null; - if (command.DescribeForPreparation) - { - var preparation = await command.ReadPreparationDescriptionAsync( - decoder, context.GetProtocolStatic().RowDescription) - .ConfigureAwait(false); - error = preparation.Item1; - preparationParameterTypes = preparation.Item2; - requestedRowDescription = preparation.Item3; - } - else if (hasPreparedDescription && !describeOnly) - { - // Prepared commands with a known description have the compact BindComplete -> - // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes this - // frame rather than a nested parser coroutine. - if (!decoder.TryMoveNext()) - { - if (!await decoder.MoveNextAsync().ConfigureAwait(false)) - decoder.ThrowUnexpectedEof(); - } - var message = decoder.Current; - if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) - { - error = bindError; - } - else - { - if (!decoder.TryMoveNext()) - { - if (!await decoder.MoveNextAsync().ConfigureAwait(false)) - decoder.ThrowUnexpectedEof(); - } - decoder.Current.DebugEnsureExpected(PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); - error = null; - } - requestedRowDescription = null; - } - else - { - (error, requestedRowDescription) = await command - .ReadUntilExecuteAsync(decoder, context.GetProtocolStatic().RowDescription) - .ConfigureAwait(false); - } - return InitializeResult( - commandIndex, error, requestedRowDescription, preparationParameterTypes); - } - - CommandResult ReadResult(int commandIndex) - { - var context = _state.Context; - var decoder = context.Decoder; - if (context.IsProtocolClosed) - throw context.FlowTerminationException; - ref readonly var command = ref _state.Commands.ItemRef(commandIndex); - decoder.UseReadTimeout(command.Timeout); - PgError? error; - RowDescription? requestedRowDescription; - ParameterTypeList? preparationParameterTypes = null; - if (command.DescribeForPreparation) - { - var preparation = command.ReadPreparationDescription( - decoder, context.GetProtocolStatic().RowDescription); - error = preparation.Item1; - preparationParameterTypes = preparation.Item2; - requestedRowDescription = preparation.Item3; - } - else - { - (error, requestedRowDescription) = command.ReadUntilExecute( - decoder, context.GetProtocolStatic().RowDescription); - } - return InitializeResult( - commandIndex, error, requestedRowDescription, preparationParameterTypes); - } - - CommandResult InitializeResult( - int commandIndex, PgError? error, RowDescription? requestedRowDescription, - ParameterTypeList? preparationParameterTypes = null) - { - var context = _state.Context; - ref readonly var readState = ref context.GetProtocolStatic(); - ref readonly var command = ref _state.Commands.ItemRef(commandIndex); - readState.ResultMessageEnumerator.Initialize(command, context.Decoder); - var result = readState.CommandResult; - var descriptor = command.Descriptor; - // A named unprepared statement that parsed becomes a prepared descriptor. - if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault - && (error is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) - { - descriptor = CommandDescriptor.CreatePrepared(descriptor.CommandName, - preparationParameterTypes ?? descriptor.ParameterTypes, - requestedRowDescription?.Preserve()); - } - result.Initialize(_ops.Flow, commandIndex, descriptor, requestedRowDescription, - !command.DescribeOnly, command.IsSimple(), error); - _ops.OnCommandResult(result); - return result; - } - - async ValueTask<(PgError Error, TransactionStatus TransactionStatus)?> CompleteCurrentResultAsync() - { - var enumerator = _state.Context.GetProtocolStatic().ResultMessageEnumerator; - await enumerator.DisposeAsync().ConfigureAwait(false); - return enumerator.CompleteError; - } - - (PgError Error, TransactionStatus TransactionStatus)? CompleteCurrentResult() - { - var enumerator = _state.Context.GetProtocolStatic().ResultMessageEnumerator; - enumerator.Dispose(); - return enumerator.CompleteError; - } - - async ValueTask SkipDiscardedCommandsAsync() - { - while (++_state.CommandIndex < _state.Commands.Count && !_state.Commands[_state.CommandIndex].WithSync) { } - await ReadRfqAsync().ConfigureAwait(false); - if (_state.CommandIndex == _state.Commands.Count) - _state.ReadFlowRfq = false; - } - - void SkipDiscardedCommands() - { - while (++_state.CommandIndex < _state.Commands.Count && !_state.Commands[_state.CommandIndex].WithSync) { } - ReadRfq(); - if (_state.CommandIndex == _state.Commands.Count) - _state.ReadFlowRfq = false; - } - - async ValueTask ReadRfqAsync() - { - var message = await _state.Context.Decoder.GetNextAsync().ConfigureAwait(false); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - - void ReadRfq() - { - var message = _state.Context.Decoder.GetNext(); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - - async ValueTask CompleteBatchAsync() - { - if (_state.ReadFlowRfq) - await ReadRfqAsync().ConfigureAwait(false); - await DisposeRegistrationsAsync().ConfigureAwait(false); - Finish(); - } - - void CompleteBatch() - { - if (_state.ReadFlowRfq) - ReadRfq(); - DisposeRegistrations(); - Finish(); - } - - // The wire is at this command's RFQ. Release the shared read objects, record the outcome the - // consumer must observe, then complete the pipeline task. - void Finish() - { - _state.Context.GetProtocolStatic().Reset(); - _state.Current = null; - _state.CurrentPublished = false; - if (IsCancelRequested) - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, - new OperationCanceledException(_state.ColdState!.DeliverToken), null); - else if (Volatile.Read(ref _state.ColdState)?.CloseException is { } close) - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, close, null); - CompletePipelineTask(null); - } - - bool IsOwnCancellation(PgError error) - => IsCancelRequested && error.SqlState == PgErrorCodes.QueryCanceled; - - // The owning frame failed. The read error is the pipeline task's failure, which the framework - // recovers or drains. Later consumer calls replay it. - void FaultFromOwner(Exception exception) - { - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - if (Volatile.Read(ref _state.Phase) == PhaseCompleted) - return; - DisposeRegistrations(); - _state.Current = null; - _state.CurrentPublished = false; - if (HasDecoder) - _state.Context.GetProtocolStatic().Reset(); - CompletePipelineTask(exception); - } - - // Takes an idle decoder for a drain. Returns false when a frame already owns it, which will observe - // the latch itself, or when the flow reached its terminal. - bool TryTakeOverDrain() - { - while (true) - { - var phase = Volatile.Read(ref _state.Phase); - if (phase is PhaseInitial && !HasDecoder) - return false; - if (phase is not (PhaseInitial or PhaseResultReady)) - return false; - if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) - continue; - NotifyDrainStarted(); - _state.ConsumerDetached = true; - ThreadPool.UnsafeQueueUserWorkItem(static state => - _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), - _ops.Flow); - return true; - } - } - - void NotifyDrainStarted() - { - if (Interlocked.Exchange(ref _state.DrainStarted, 1) is 0) - _ops.OnDrainStarted(); - } - - // Autonomous drain. Owns the decoder until the pipeline task completes. Never throws. - async ValueTask DrainAsync() - { - try - { - var result = _state.Current; - if (result is null) - { - await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); - if (_state.CommandIndex < 0) - _state.CommandIndex = 0; - if (_state.CommandIndex >= _state.Commands.Count) - { - await CompleteBatchAsync().ConfigureAwait(false); - return; - } - result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); - } - - while (true) - { - var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); - CaptureDrainError(result, completeError); - _state.CurrentPublished = false; - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) - await SkipDiscardedCommandsAsync().ConfigureAwait(false); - if (++_state.CommandIndex >= _state.Commands.Count) - break; - result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); - } - await CompleteBatchAsync().ConfigureAwait(false); - } - catch (TimeoutException ex) - { - // A timeout during semantic drain escalates the same cancellation episode immediately. - // The pipeline failure then hands any remaining wire obligation to recovery. - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, - BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); - FaultFromOwner(ex); - } - catch (Exception ex) - { - FaultFromOwner(ex); - } - } - - // A read timeout is terminal for the consumer but not for the wire obligation. This frame has - // released the decoder read tenure, so transfer ownership to an autonomous semantic drain while - // the original MoveNext returns its timeout immediately. - void HandleReadTimeout(TimeoutException exception) - { - Interlocked.CompareExchange( - ref GetOrCreateColdState().TerminalException, exception, null); - _state.ConsumerDetached = true; - NotifyDrainStarted(); - Interlocked.Exchange(ref _state.Phase, PhaseDraining); - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, - BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); - ThreadPool.UnsafeQueueUserWorkItem(static state => - _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), - _ops.Flow); - } - - void Drain() - { - try - { - var result = _state.Current; - if (result is null) - { - // Synchronous disposal before any read. The activation bridge normally completed long - // ago, so this bridge is rarely more than a status check. - new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).AsTask().GetAwaiter().GetResult(); - if (_state.CommandIndex < 0) - _state.CommandIndex = 0; - if (_state.CommandIndex >= _state.Commands.Count) - { - CompleteBatch(); - return; - } - result = ReadResult(_state.CommandIndex); - } - - while (true) - { - var completeError = CompleteCurrentResult(); - CaptureDrainError(result, completeError); - _state.CurrentPublished = false; - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) - SkipDiscardedCommands(); - if (++_state.CommandIndex >= _state.Commands.Count) - break; - result = ReadResult(_state.CommandIndex); - } - CompleteBatch(); - } - catch (Exception ex) - { - FaultFromOwner(ex); - } - } - - internal ValueTask DisposeAsync() - { - while (true) - { - var phase = Volatile.Read(ref _state.Phase); - switch (phase) - { - case PhaseInitial: - case PhaseResultReady: - if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) - continue; - NotifyDrainStarted(); - _state.ConsumerDetached = true; - if (_state.Current is { IsComplete: false } - || _state.CommandIndex + 1 < _state.Commands.Count) - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); - return _state.WaitForDrainOnDispose ? DisposeDrainAsync() : FireAndForgetDrain(); - case PhaseReading: - _state.ConsumerDetached = true; - NotifyDrainStarted(); - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); - return _state.WaitForDrainOnDispose ? DisposeCompletedAsync() : default; - default: - return !_state.WaitForDrainOnDispose || _state.ConsumerObservedCompletion - ? default - : DisposeCompletedAsync(); - } - } - } - - // Drains on the disposer's frame, then waits for framework release so a drain error can surface. - async ValueTask DisposeDrainAsync() - { - await DrainAsync().ConfigureAwait(false); - await DisposeCompletedAsync().ConfigureAwait(false); - } - - ValueTask FireAndForgetDrain() - { - _ = DrainAsync(); - return default; - } - - async ValueTask DisposeCompletedAsync() - { - await WaitForCompletionAsync().ConfigureAwait(false); - ThrowDrainErrors(); - } - - // Flow completion is independent of errors accumulated while draining. A close is a clean - // terminal for a disposing consumer. - async ValueTask WaitForCompletionAsync() - { - try - { - await _ops.Flow.WaitForComplete().ConfigureAwait(false); - } - catch (PgClientClosedException) - { - } - } - - internal void Dispose() - { - if (!_ops.IsAsyncAtDispatch) - EnsureSyncHandoff(); - while (true) - { - var phase = Volatile.Read(ref _state.Phase); - switch (phase) - { - case PhaseInitial: - case PhaseResultReady: - if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) - continue; - NotifyDrainStarted(); - _state.ConsumerDetached = true; - if (_state.Current is { IsComplete: false } - || _state.CommandIndex + 1 < _state.Commands.Count) - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); - Drain(); - if (_state.WaitForDrainOnDispose) - DisposeCompleted(); - return; - case PhaseReading: - _state.ConsumerDetached = true; - NotifyDrainStarted(); - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); - if (_state.WaitForDrainOnDispose) - DisposeCompleted(); - return; - default: - if (_state.WaitForDrainOnDispose && !_state.ConsumerObservedCompletion) - DisposeCompleted(); - return; - } - } - } - - void DisposeCompleted() - { - try - { - _ops.Flow.WaitForCompleteSynchronously(); - } - catch (PgClientClosedException) - { - } - ThrowDrainErrors(); - } - - void CaptureDrainError(CommandResult result, - (PgError Error, TransactionStatus TransactionStatus)? completeError) - { - if (!_state.ConsumerDetached || _state.CurrentPublished) - return; - var error = result.Error ?? completeError?.Error; - if (error is null || IsOwnCancellation(error)) - return; - var cold = GetOrCreateColdState(); - (cold.DrainErrors ??= new()).Add(PgErrorException.Create(error)); - } - - void ThrowDrainErrors() - { - if (Volatile.Read(ref _state.ColdState)?.DrainErrors is not { Count: > 0 } errors) - return; - if (errors.Count is 1) - ExceptionDispatchInfo.Throw(errors[0]); - throw new AggregateException(errors); - } - - // When true, disposal waits for the drain to reach RFQ and for framework release. Otherwise it - // returns while the drain continues autonomously. - void RegisterCancellation(CancellationToken callerToken) - { - // Keep the second token/registration pair off ordinary flow objects. Default-token traffic - // does not need cancellation state at all and remains the allocation/footprint hot path. - var cancellation = Volatile.Read(ref _state.ColdState); - if (callerToken.CanBeCanceled || cancellation is not null) - { - cancellation ??= GetOrCreateColdState(); - if (callerToken != cancellation.CallerToken) - { - var callerRegistration = cancellation.CallerRegistration; - cancellation.CallerRegistration = default; - callerRegistration.Dispose(); - cancellation.CallerToken = callerToken; - } - if (callerToken.CanBeCanceled && cancellation.CallerRegistration == default) - cancellation.CallerRegistration = callerToken.UnsafeRegister(static (state, token) - => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( - token, CommandExecutionCancellationScope.CurrentWindow), _ops.Flow); - } - - if (_state.FlowToken.CanBeCanceled && _state.FlowRegistration == default) - _state.FlowRegistration = _state.FlowToken.UnsafeRegister(static (state, token) - => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( - token, CommandExecutionCancellationScope.RemainingFlow), _ops.Flow); - } - - ValueTask DisposeRegistrationsAsync() - { - var cancellation = Volatile.Read(ref _state.ColdState); - var callerRegistration = cancellation?.CallerRegistration ?? default; - if (callerRegistration == default && _state.FlowRegistration == default) - return default; - if (cancellation is not null) - cancellation.CallerRegistration = default; - var flowRegistration = _state.FlowRegistration; - _state.FlowRegistration = default; - return DisposeRegistrationsAsync(callerRegistration, flowRegistration); - - static async ValueTask DisposeRegistrationsAsync( - CancellationTokenRegistration callerRegistration, - CancellationTokenRegistration flowRegistration) - { - await callerRegistration.DisposeAsync().ConfigureAwait(false); - await flowRegistration.DisposeAsync().ConfigureAwait(false); - } - } - - void DisposeRegistrations() - { - var cancellation = Volatile.Read(ref _state.ColdState); - var callerRegistration = cancellation?.CallerRegistration ?? default; - if (cancellation is not null) - cancellation.CallerRegistration = default; - var flowRegistration = _state.FlowRegistration; - _state.FlowRegistration = default; - callerRegistration.Dispose(); - flowRegistration.Dispose(); - } - - // Cancellation only latches intent and requests a backend cancel. The frame owning the decoder - // delivers it after the wire is back at RFQ. An idle flow drains autonomously first. - void RequestCancel(CancellationToken token, CommandExecutionCancellationScope scope, - BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, - BackendCancellationTiming subsequentTiming = BackendCancellationTiming.AfterGrace) - { - if (Volatile.Read(ref _state.Phase) == PhaseCompleted) - return; - var cancellation = GetOrCreateColdState(); - cancellation.DeliverToken = token; - RaiseCancellationScope(cancellation, scope); - RaiseCancellationTiming(ref cancellation.Timing, timing); - RaiseCancellationTiming(ref cancellation.SubsequentTiming, subsequentTiming); - Interlocked.Exchange(ref cancellation.CancelRequested, true); - if (HasDecoder) - RequestBackendCancellation(); - TryTakeOverDrain(); - } - - static void RaiseCancellationScope(CommandExecutionColdState cancellation, CommandExecutionCancellationScope scope) - { - var requested = (int)scope; - var current = Volatile.Read(ref cancellation.Scope); - while (current < requested) - { - var observed = Interlocked.CompareExchange( - ref cancellation.Scope, requested, current); - if (observed == current) - return; - current = observed; - } - } - - static void RaiseCancellationTiming( - ref int location, BackendCancellationTiming timing) - { - var requested = (int)timing; - var current = Volatile.Read(ref location); - while (current < requested) - { - var observed = Interlocked.CompareExchange(ref location, requested, current); - if (observed == current) - return; - current = observed; - } - } - - internal Task CancelAsync() - { - var cancellation = GetOrCreateColdState(); - var delivery = Volatile.Read(ref cancellation.Delivery) - ?? Interlocked.CompareExchange(ref cancellation.Delivery, - new(TaskCreationOptions.RunContinuationsAsynchronously), null) - ?? cancellation.Delivery; - if (Volatile.Read(ref _state.Phase) is PhaseCompleted) - { - delivery.TrySetResult(); - return delivery.Task; - } - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); - if (Volatile.Read(ref _state.Phase) is PhaseCompleted) - delivery.TrySetResult(); - return delivery.Task; - } - - void RequestBackendCancellation() - { - var cancellation = GetOrCreateColdState(); - var episodeKey = Volatile.Read(ref cancellation.EpisodeKey); - if (episodeKey is null) - { - var created = new object(); - episodeKey = Interlocked.CompareExchange( - ref cancellation.EpisodeKey, created, null) ?? created; - } - _state.Context.RequestBackendCancellation( - _ops.Flow, _ops.Flow.CancellationWindow, - (BackendCancellationTiming)Volatile.Read(ref cancellation.Timing), - Volatile.Read(ref cancellation.Delivery), - episodeKey, - Math.Max(Volatile.Read(ref cancellation.Scope), (int)CommandExecutionCancellationScope.CurrentWindow), - (BackendCancellationTiming)Volatile.Read(ref cancellation.SubsequentTiming)); - } - - // Graceful stop. An unactivated flow releases its consumer, the closing wire owns its response. - // An idle activated flow drains itself to RFQ so the pipeline can complete. - internal void OnStopping(Exception exception) - { - Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); - if (CompleteReady(exception, runContinuationsAsynchronously: true)) - { - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - return; - } - TryTakeOverDrain(); - } - - // Forceful abort. No frame can read a dead wire, so an idle owner faults the pipeline task - // directly. A frame in flight fails on its own read. - internal void OnAbort(Exception exception) - { - Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); - if (CompleteReady(exception, runContinuationsAsynchronously: true)) - { - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - return; - } - while (true) - { - var phase = Volatile.Read(ref _state.Phase); - if (phase is not (PhaseInitial or PhaseResultReady)) - return; - if (Interlocked.CompareExchange(ref _state.Phase, PhaseCompleted, phase) != phase) - continue; - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - _state.PipelineTaskSource.SetException(exception, runContinuationsAsynchronously: true); - return; - } - } - - internal void Fail(Exception exception) - { - // A result callback failed on the frame that owns the decoder. Its throw propagates there. - Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); - } - - internal void OnReleasing(Exception? exception) - { - Volatile.Read(ref _state.ColdState)?.Delivery?.TrySetResult(); - DisposeRegistrations(); - _state.Commands.Return(); - } - - internal void OnDiscarded() - { - _ops.OnDiscarded(); - _state.Commands.Return(); - } - - internal void OnReset() - { - _state.Phase = PhaseInitial; - _state.CommandIndex = -1; - _state.Context = default; - _state.ContextPublished = false; - _state.Current = null; - _state.CurrentPublished = false; - _state.ReadFlowRfq = false; - _state.ConsumerDetached = false; - _state.ConsumerObservedCompletion = false; - _state.ReadySource.Reset(); - _state.PipelineTaskSource.Reset(); - _state.ReadyCompletion = 0; - _state.DrainStarted = 0; - _state.FlowToken = default; - _state.FlowRegistration = default; - _state.ColdState = null; - _state.SyncHandoffClaimed = false; - _state.HandoffEvent?.ResetInteraction(); - _state.WaitForDrainOnDispose = true; - } - -} - -internal sealed partial class CommandExecutionFlow -{ - bool IValueTaskSource.GetResult(short token) => _state.ReadySource.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _state.ReadySource.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - => _state.ReadySource.OnCompleted(continuation, state, token, flags); - - void IValueTaskSource.GetResult(short token) => _state.PipelineTaskSource.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _state.PipelineTaskSource.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - => _state.PipelineTaskSource.OnCompleted(continuation, state, token, flags); - - public readonly struct Enumerator : IAsyncEnumerator, IDisposable - { - readonly CommandExecutionFlow? _flow; - readonly CancellationToken _cancellationToken; - - public Enumerator(CommandExecutionFlow flow) - : this(flow, default) - { } - - internal Enumerator(CommandExecutionFlow flow, CancellationToken cancellationToken) - { - _flow = flow; - _cancellationToken = cancellationToken; - } - - public Enumerator GetAsyncEnumerator() => this; - - public Enumerator GetEnumerator() => this; - - public bool MoveNext() => _flow?.Core.MoveNext() ?? false; - - public ValueTask MoveNextAsync() => MoveNextAsync(_cancellationToken); - - public ValueTask MoveNextAsync(CancellationToken cancellationToken) - => _flow is null ? new(false) : _flow.Core.MoveNextAsync(cancellationToken); - - public CommandResult Current => _flow?._state.Current ?? default!; - - public ValueTask DisposeAsync() => _flow is null ? default : _flow.Core.DisposeAsync(); - - public void Dispose() => _flow?.Core.Dispose(); - } -} diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 4c8c3f3..9d6f2c5 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -6,7 +6,7 @@ namespace Slon.Pg.Protocol.Flows; -partial class CommandFlow +public sealed partial class CommandFlow { internal enum MoveNextStatus : byte { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 82850dd..72e4a7c 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -1,10 +1,10 @@ +using System.Collections; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; using System.Threading.Tasks.Sources; using Slon.Runtime.CompilerServices; -// A unique result type distinguishes the caller gate from this flow's other IValueTaskSource instantiations. -using FlowCallerInteractionCoreResult = System.ValueTuple; namespace Slon.Pg.Protocol.Flows; @@ -12,7 +12,8 @@ namespace Slon.Pg.Protocol.Flows; public abstract class CommandFlowObserver : PgClientFlowObserver { protected internal virtual void OnStarted(CommandFlow flow, object? state) { } - protected internal virtual void OnCommandResult(CommandFlow flow, CommandResult result, object? state) { } + protected internal virtual void OnCommandResult( + CommandFlow flow, CommandResult result, object? state) { } protected internal virtual void OnDrainStarted(CommandFlow flow, object? state) { } } @@ -26,1306 +27,1502 @@ public readonly struct CommandFlowOptions public TimeSpan? PendingTimeout { get; init; } } +internal sealed class CommandExecutionColdState +{ + internal bool CancelRequested; + internal CancellationToken CallerToken; + internal CancellationTokenRegistration CallerRegistration; + internal CancellationToken DeliverToken; + internal int Scope; + internal int Timing; + internal int SubsequentTiming; + internal TaskCompletionSource? Delivery; + internal object? EpisodeKey; + internal Exception? CloseException; + // Replayed by later consumer calls once the flow reached its terminal. + internal Exception? TerminalException; + // Command errors observed while draining without a consumer. Multiple Sync windows may each + // produce an independent ErrorResponse, all of which belong to the waiting disposer. + internal List? DrainErrors; +} + +internal enum CommandExecutionCancellationScope : byte +{ + CurrentWindow = 1, + RemainingFlow = 2 +} + +// Mutable execution state is stored inline by each concrete host. Core algorithms re-enter this +// field through their host-specific ops value after every await; they never mutate a copied struct. +[StructLayout(LayoutKind.Auto)] +internal struct CommandExecutionState +{ + internal int Phase; + internal CommandList Commands; + internal TimeSpan? PendingTimeout; + internal int CommandIndex; + internal PgClientFlow.Context Context; + internal bool ContextPublished; + internal CommandResult? Current; + internal bool CurrentPublished; + internal bool ReadFlowRfq; + internal bool ConsumerDetached; + internal bool ConsumerObservedCompletion; + internal Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore ReadySource; + internal int ReadyCompletion; + internal Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore PipelineTaskSource; + internal CancellationToken FlowToken; + internal CancellationTokenRegistration FlowRegistration; + internal CommandExecutionColdState? ColdState; + internal FlowHandoffEvent? HandoffEvent; + internal bool SyncHandoffClaimed; + internal int DrainStarted; + internal bool EnableActivationTimeout; + internal bool WaitForDrainOnDispose; +} + +/// Executes an ordered command list with consumer-owned synchronous or asynchronous result decoding. [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] -public partial class CommandFlow : PgClientFlow, IValueTaskSource, IValueTaskSource, IValueTaskSource +public sealed partial class CommandFlow : PgClientFlow, IValueTaskSource, IValueTaskSource { - sealed class PreparationReadState + static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); + CommandExecutionState _state; + CommandFlowObserver? _commandObserver; + object? _commandObserverState; + + CommandFlow(bool async, TimeSpan? pendingTimeout = null) + : base(supportsDeferredFlush: true) { - internal ParameterTypeList ParameterTypes; + _state.CommandIndex = -1; + _state.EnableActivationTimeout = true; + _state.WaitForDrainOnDispose = true; + _state.PendingTimeout = pendingTimeout; + IsAsync = async; + if (!async) + _state.HandoffEvent = new(false); } - static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); + public CommandFlow(bool async, params ReadOnlySpan commands) + : this(async) + => Initialize(async, commands); - internal override bool DefersSyncHandoff => true; + internal CommandFlow( + bool async, bool enableActivationTimeout, params ReadOnlySpan commands) + : this(async, commands) + => _state.EnableActivationTimeout = enableActivationTimeout; - internal enum CancellationScope : byte - { - None, - CurrentWindow = 1, - RemainingFlow = 2 - } - - // Flow state - CommandList _commands; - TimeSpan? _pendingTimeout; - FlowCallerInteractionCore _callerInteractionCore; - // Cancellation is cold; keep its tokens, registrations and attribution state off ordinary flows. - CancellationState? _cancellationState; - // Errors encountered while draining are surfaced by a waiting DisposeAsync. Live consumers observe - // their errors directly, and the list remains unallocated on the successful path. - List? _drainErrors; - // Set only by ConsumeNonQueryAsync, after enqueue but before any consumer-side gate release. - // The body cannot reach first publication until such a release, and every release publishes - // this write, so plain accesses suffice and the body observes the mode at first wake. - bool _consumeNonQuery; - bool IsConsumingNonQuery => _consumeNonQuery; - bool IsConsumingAutonomously => IsDraining || IsConsumingNonQuery; - long _nonQueryRecordsAffected; - - // Once draining, the body bypasses result handoffs and reads autonomously to RFQ. This is state, not - // an I/O cancellation token: canceling the I/O would prevent restoration of a clean wire boundary. - bool _draining; - internal bool IsDraining => Volatile.Read(ref _draining); - // Body-thread-only guard: later commands must not change the drive mode chosen on drain entry. - bool _drainModeEntered; - - - // Distinguishes consumer disposal from a body-initiated drain; disposal suppresses terminal OCE delivery. - bool _consumerDisposed; - // When true, DisposeAsync awaits the body's drain to RFQ. Otherwise it returns while the body drains; - // pipeline retirement still prevents the next flow from observing a dirty wire. - internal bool WaitForDrainOnDispose { get; set; } = true; - - // Consumer disposal without waiting for the autonomous drain. - void MarkConsumerGone() - { - Volatile.Write(ref _consumerDisposed, true); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, - BackendCancellationTiming.AtReadFrontier); - } - - // Consumer disposal while waiting for the autonomous drain. - void MarkConsumerWaitForDrain() - { - Volatile.Write(ref _consumerDisposed, true); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, - BackendCancellationTiming.AtReadFrontier); - } - - // A synchronous consumer cannot abandon its drive obligation while the body is parked. Give the - // cancellation path a delivery source so WakeBody transfers that obligation to the dedicated driver. - void MarkSyncConsumerGone() - { - Volatile.Write(ref _consumerDisposed, true); - _ = GetOrCreateCancelDelivery(); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, - BackendCancellationTiming.AtReadFrontier); - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); - _callerInteractionCore.WakeBody(useDedicatedDriver: true); - } - - // Enumeration already ended; only transfer the live body tail to autonomous ownership. - // There is no unfinished consumer work to justify a backend cancellation intent. - void MarkTerminalConsumerGone() - { - Volatile.Write(ref _consumerDisposed, true); - Volatile.Write(ref _draining, true); - } - - // Result publication orders these body-owned fields before the consumer can observe the - // CommandResult. IsComplete is consumer-owned after that handoff. A behavior-limited reader may - // consider its visible result final while later commands still exist, so use the physical - // command index rather than an ADO-visible result count. - bool IsFullyConsumedFinalResult - => _isResultReady && _commandIndex >= CommandCount - 1 - && _enumeratorCurrent is { IsComplete: true }; - - // A body-initiated drain keeps the consumer attached for terminal cancellation or close delivery. - void MarkBodyInitiatedDrain() => Volatile.Write(ref _draining, true); - - // Result-production state - RowDescription? _requestedRowDescription; - PgError? _pgError; - int _commandIndex = -1; - PgDecoder? _decoder; - bool _readFlowRfq; - - // Pipelined dispatch state. Lives here (not on PgClientFlow base) because the shared-promise - // optimization that needs these fields is CommandFlow-specific (see DispatchPipelinedRead). - Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _executePipelinedCore; - Context _context; - bool _contextPublished; - // Consumer terminality, body terminality, and framework release are distinct phases. Start and - // pre-start termination race during shutdown, so one atomic state owns that decision. - const int BodyNotStarted = 0; - const int BodyRunning = 1; - const int BodyTerminated = 2; - int _bodyState; - DetachedPublication _detachedPublication; - - enum DetachedPublication : byte - { - None, - Result, - Completion - } - - sealed class CancellationState - { - internal CancellationToken CallerToken; - internal CancellationToken FlowToken; - internal CancellationTokenRegistration CallerRegistration; - internal CancellationTokenRegistration FlowRegistration; - internal bool Requested; - internal int Scope; - internal int Timing; - internal int SubsequentTiming; - internal CancellationToken DeliverToken; - internal TaskCompletionSource? Delivery; - internal object? EpisodeKey; - internal bool DeliverOce; - - internal void Reset() + internal CommandFlow(bool async, CommandList commands, TimeSpan? pendingTimeout = null) + : this(async, pendingTimeout) + => Initialize(async, new CommandFlowOptions { - CallerToken = default; - FlowToken = default; - CallerRegistration.Dispose(); - CallerRegistration = default; - FlowRegistration.Dispose(); - FlowRegistration = default; - Requested = false; - Scope = (int)CancellationScope.None; - Timing = (int)BackendCancellationTiming.AfterGrace; - SubsequentTiming = (int)BackendCancellationTiming.AfterGrace; - DeliverToken = default; - Delivery = null; - EpisodeKey = null; - DeliverOce = false; - } - } - CommandFlow() : base(supportsDeferredFlush: true) - { - _callerInteractionCore.Initialize(); - } - - // Interactive commands carry caller patience, so arm the activation timeout. - protected override bool EnableActivationTimeout => true; - protected override TimeSpan? PendingTimeout => _pendingTimeout; - internal override TimeSpan? BackendCancellationGracePeriod - => Volatile.Read(ref _consumerDisposed) ? ConsumerDrainCancellationGracePeriod : null; + Commands = commands, + PendingTimeout = pendingTimeout + }); - public CommandFlow(bool async, params ReadOnlySpan commands) : this() - => Initialize(async, commands); - public CommandFlow(bool async, in CommandFlowOptions options) : this() + public CommandFlow(bool async, in CommandFlowOptions options) + : this(async, options.PendingTimeout) => Initialize(async, options); - private protected CommandFlow(bool async, TimeSpan? pendingTimeout) : this() - { - IsAsync = async; - _pendingTimeout = pendingTimeout; - } - public CommandFlow Initialize(bool async, params ReadOnlySpan commands) - => Initialize(async, options: new() { Commands = new(commands) }); + => Initialize(async, new CommandFlowOptions { Commands = new(commands) }); public CommandFlow Initialize(bool async, in CommandFlowOptions options) { IsAsync = async; + if (!async) + _state.HandoffEvent ??= new(false); + var commands = options.Commands; + if (commands.Count is 0) + return this; + _state.Commands = commands; + _state.PendingTimeout = options.PendingTimeout; + _commandObserver = options.Observer; + _commandObserverState = options.ObserverState; if (options.Observer is { } observer) - SetObserver(observer, options.ObserverState); - _commands = options.Commands; - _pendingTimeout = options.PendingTimeout; - options.Observer?.OnStarted(this, options.ObserverState); - return this; - } - - // Declares internal non-query consumption and runs it to completion. The single entry point - // makes the ownership rule structural: no enumerator is exposed on this path, so mixing - // enumeration with internal consumption is unrepresentable. The declaration precedes any - // consumer-side gate release, so the body observes it at first wake and never publishes. - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - internal async ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) - { - _consumeNonQuery = true; - _nonQueryRecordsAffected = -1; - var enumerator = GetAsyncEnumerator(cancellationToken); - try - { - // Release a body parked pre-publication and wake the flow. - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); - _callerInteractionCore.WakeBody(); - _ = await EnumeratorMoveNextTask.ConfigureAwait(false); - // Internal consumption owns error delivery. Errors collected during the internal - // drain have no other outlet: DisposeAsync deliberately skips a completed flow. - if (_drainErrors is { Count: > 0 } errors) - throw errors.Count == 1 ? errors[0] : new AggregateException(errors); - return _nonQueryRecordsAffected; - } - finally { - await enumerator.DisposeAsync().ConfigureAwait(false); + SetObserver(observer, options.ObserverState); + observer.OnStarted(this, options.ObserverState); } + return this; } - // The token in force for the current read: the flow token once fired (whole-flow cancel), else the - // per-read token if cancelable, else the flow token. - CancellationToken EffectiveCancellationToken - => GetEffectiveCancellationToken(Volatile.Read(ref _cancellationState)); - - static CancellationToken GetEffectiveCancellationToken(CancellationState? cancellation) - => cancellation is null ? default - : cancellation.FlowToken.IsCancellationRequested ? cancellation.FlowToken - : cancellation.CallerToken.CanBeCanceled ? cancellation.CallerToken - : cancellation.FlowToken; - - public int CommandCount => _commands.Count; - internal virtual int VisibleCommandCount => _commands.VisibleCount; - public bool IsResultReady => _isResultReady; - - public Enumerator GetEnumerator() - { - return new Enumerator(this); - } + internal override bool DefersSyncHandoff => true; + private protected override FlowHandoffEvent? HandoffEvent => _state.HandoffEvent; + protected override bool EnableActivationTimeout => _state.EnableActivationTimeout; + protected override TimeSpan? PendingTimeout => _state.PendingTimeout; + internal override TimeSpan? BackendCancellationGracePeriod + => Volatile.Read(ref _state.ConsumerDetached) + ? ConsumerDrainCancellationGracePeriod + : null; - // Bind at submission because eager writing precedes the first MoveNextAsync. internal override void BindCallerToken(CancellationToken cancellationToken) - => GetOrCreateCancellationState().FlowToken = cancellationToken; + => _state.FlowToken = cancellationToken; internal override CancellationToken MigrationCancellationToken - => Volatile.Read(ref _cancellationState)?.FlowToken ?? default; + => _state.FlowToken; + + public Enumerator GetEnumerator() + => new(this, default); public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { - // A missing per-call token must not replace the flow token captured at submission. + // A missing enumeration token must not erase the token captured when the flow was queued. if (cancellationToken.CanBeCanceled) - GetOrCreateCancellationState().FlowToken = cancellationToken; - return new(this); + _state.FlowToken = cancellationToken; + return new(this, cancellationToken); + } + + internal CommandResult? CurrentResult => _state.Current; + public int CommandCount => _state.Commands.Count; + public bool IsResultReady => Core.IsResultReady; + internal int VisibleCommandCount => _state.Commands.VisibleCount; + internal ValueTask MoveNextResultAsync(CancellationToken cancellationToken) + => Core.MoveNextAsync(cancellationToken); + internal void DisposeResults() => Core.Dispose(); + internal ValueTask DisposeResultsAsync() => Core.DisposeAsync(); + internal bool WaitForDrainOnDispose + { + get => _state.WaitForDrainOnDispose; + set => _state.WaitForDrainOnDispose = value; } + CommandExecutionCore Core => new(new(this)); + + internal ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) + => Core.ConsumeNonQueryAsync(cancellationToken); + protected override ValueTask ExecuteAuto(Context context) + => Core.ExecuteAuto(context); + + internal Task CancelAsync() => Core.CancelAsync(); + + internal override bool ResetsSharedReadStateBeforeRelease => true; + protected override void OnStopping(Exception exception) => Core.OnStopping(exception); + protected override void OnAbort(Exception exception) => Core.OnAbort(exception); + internal override void Fail(Exception exception) => Core.Fail(exception); + protected override void OnReleasing(Exception? exception) => Core.OnReleasing(exception); + protected override void OnDiscarded() => Core.OnDiscarded(); + protected override void OnReset() => Core.OnReset(); + + readonly struct Ops(CommandFlow owner) : ICommandExecutionFlowOps { - if (!IsAsync && _callerInteractionCore.IsWaiting) - return ExecuteAfterHandoff(context); + readonly CommandFlow _owner = owner; - return new(ExecuteAutoCore(context)); + public static Ops Create(PgClientFlow flow) => new((CommandFlow)flow); + public PgClientFlow Flow => _owner; + public ref CommandExecutionState State => ref _owner._state; + public bool IsAsync + { + get => _owner.IsAsync; + set => _owner.IsAsync = value; + } + public bool IsAsyncAtDispatch => _owner.IsAsyncAtDispatch; + public bool HasSuccessfulActivation => _owner.HasSuccessfulActivation; + public void WaitForSyncHandoff() => _owner.WaitForSyncHandoff(); + public void OnCommandResult(CommandResult result) + => _owner._commandObserver?.OnCommandResult( + _owner, result, _owner._commandObserverState); + public void OnDrainStarted() + => _owner._commandObserver?.OnDrainStarted( + _owner, _owner._commandObserverState); + public void OnDiscarded() + => _owner.GetObserver(out var observerState)?.OnCompleting( + _owner, null, observerState); } +} + +internal interface ICommandExecutionFlowOps + where TSelf : struct, ICommandExecutionFlowOps +{ + static abstract TSelf Create(PgClientFlow flow); + PgClientFlow Flow { get; } + ref CommandExecutionState State { get; } + bool IsAsync { get; set; } + bool IsAsyncAtDispatch { get; } + bool HasSuccessfulActivation { get; } + void WaitForSyncHandoff(); + void OnCommandResult(CommandResult result); + void OnDrainStarted(); + void OnDiscarded(); +} + +readonly struct CommandExecutionCore(TOps ops) + where TOps : struct, ICommandExecutionFlowOps +{ + const int PhaseInitial = 0; + const int PhaseReading = 1; + const int PhaseResultReady = 2; + const int PhaseDraining = 3; + const int PhaseCompleted = 4; + + readonly TOps _ops = ops; + ref CommandExecutionState _state => ref _ops.State; + internal bool IsResultReady => Volatile.Read(ref _state.Phase) is PhaseResultReady; [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask ExecuteAfterHandoff(Context context) + internal async ValueTask ConsumeNonQueryAsync( + CancellationToken cancellationToken = default) { + var recordsAffected = -1L; + if (cancellationToken.CanBeCanceled) + _state.FlowToken = cancellationToken; try { - await YieldToCaller(); + while (await MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + var result = _state.Current!; + await result.CompleteAsync().ConfigureAwait(false); + var affected = result.GetCommandComplete().BatchRecordsAffected; + if (affected >= 0) + recordsAffected = recordsAffected < 0 + ? affected + : checked(recordsAffected + affected); + } + return recordsAffected; } - catch (Exception ex) + finally { - TerminateBodyBeforeStart(); - CompleteEnumerationWithException(ex); - throw; + await DisposeAsync().ConfigureAwait(false); } - - return ExecuteAutoCore(context); } - FlowTasks ExecuteAutoCore(Context context) + CommandExecutionColdState GetOrCreateColdState() + => Volatile.Read(ref _state.ColdState) ?? + Interlocked.CompareExchange(ref _state.ColdState, new(), null) ?? _state.ColdState; + + bool IsClosed => Volatile.Read(ref _state.ColdState)?.CloseException is not null; + bool IsCancelRequested => Volatile.Read(ref _state.ColdState) is { CancelRequested: true }; + bool HasDecoder => _state.ContextPublished && _ops.HasSuccessfulActivation; + + internal ValueTask ExecuteAuto(PgClientFlow.Context context) { - _context = context; - Volatile.Write(ref _contextPublished, true); - if (Volatile.Read(ref _cancellationState) is { } cancellation) - { - if (cancellation.FlowToken.IsCancellationRequested) - RequestCancel(cancellation.FlowToken, CancellationScope.RemainingFlow); - else if (Volatile.Read(ref cancellation.Requested)) - RequestBackendCancellation((BackendCancellationTiming)Volatile.Read(ref cancellation.Timing), - Volatile.Read(ref cancellation.Delivery)); - } + _state.Context = context; + _state.ContextPublished = true; ValueTask writeTask; try { - // Writes are independent of consumer admission. Inter-result gates provide backpressure. - // Async writes use transport completion; sync writes use the resumable non-blocking path so - // the caller thread retains execution ownership across readiness waits. - var appendSync = !_commands[CommandCount - 1].WithSync; - _readFlowRfq = appendSync; - // Caller cancellation never cancels wire I/O. A partially cancelled write requires - // protocol recovery and can strand already-pipelined successors; the body instead - // observes the latched intent and drains every written command to RFQ. - writeTask = IsAsync - ? _commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) + ref readonly var template = ref _state.Commands.ItemRef(_state.Commands.Count - 1); + var appendSync = !template.WithSync; + _state.ReadFlowRfq = appendSync; + // Caller cancellation never cancels wire I/O. The consumer observes the latched intent and + // drains its command to RFQ instead. + writeTask = _ops.IsAsync + ? _state.Commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) : WriteCommandsResumable(context, appendSync); - // Observe synchronous faults here; pending writes remain the framework-owned trailing task. if (writeTask.IsCompleted) writeTask.GetAwaiter().GetResult(); } catch (Exception ex) { - TerminateBodyBeforeStart(); - CompleteEnumerationWithException(ex); + // The framework recovers the wire from this throw. Only the consumer needs the fault. + FaultReady(ex); throw; } - // Read and write run concurrently; the framework observes the trailing write before releasing - // the flow, preserving single-writer tenure without blocking reads behind socket backpressure. - return new FlowTasks( - trailingExecutionTask: writeTask, - pipelineTask: DispatchPipelinedRead( - context, context.GetProtocolStatic().Promise)); + // Activation may precede or follow execution. Bridging it here guarantees a consumer resumes + // against a published context, and delivers an activation fault to the pipeline task when no + // consumer ever arrives. + var activation = context.GetDecoderAsync().ConfigureAwait(false); + if (activation.IsCompleted) + OnActivationSettled(onExecutorStrand: true); + else + activation.UnsafeOnCompleted(static state => + new CommandExecutionCore(TOps.Create((PgClientFlow)state!)) + .OnActivationSettled(onExecutorStrand: false), _ops.Flow); + return new(new FlowTasks(writeTask, new ValueTask((IValueTaskSource)_ops.Flow, _state.PipelineTaskSource.Version))); } [MethodImpl(MethodImplOptions.NoInlining)] - ValueTask WriteCommandsResumable(Context context, bool appendSync) + ValueTask WriteCommandsResumable(PgClientFlow.Context context, bool appendSync) { var encoder = context.GetEncoder(); ValueTask writeTask; using (encoder.BeginResumableWriteScope()) - writeTask = _commands.WriteCommandsResumable(encoder, appendSync); + writeTask = _state.Commands.WriteCommandsResumable(encoder, appendSync); return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); } - // Defer state-machine creation until activation because all flows share one protocol-static promise. - ValueTask DispatchPipelinedRead(Context context, ValueTaskSourcePromise promise) + // Runs on the executor strand when activation already settled, else on the activation dispatch. + // The executor strand never runs consumer code. An activation dispatch is a detached work item + // whose only remaining work is this wake, so the consumer may continue on it directly. + void OnActivationSettled(bool onExecutorStrand) { - // The shared promise may be tenured only after successful decoder activation. - var waiter = context.GetDecoderAsync().ConfigureAwait(false); - if (waiter.IsCompleted) + try { - // Only successful activation owns the shared promise. A settled fault belongs to this flow's - // private completion source because it never claimed the wire. - if (!waiter.IsCompletedSuccessfully) - { - // Preserve the activation's close, timeout, or cancellation identity. - try { waiter.GetAwaiter().GetResult(); } - catch (Exception ex) { _executePipelinedCore.SetException(ex); } - return new ValueTask(this, _executePipelinedCore.Version); - } - // Handing the shared-promise-backed task to the framework is safe: the contract guarantees - // the waiter is consumed (releasing the promise tenure via GetResult's Reset) before the - // item's position is republished, so a successor's dispatch always finds the tenure released. - PromiseAsyncValueTaskMethodBuilder.Promise = promise; - try - { - return ExecutePipelined(context); - } - finally - { - PromiseAsyncValueTaskMethodBuilder.Promise = null; - } + _ = _state.Context.GetDecoderAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + } + catch (Exception fault) + { + // Preserve the activation's close, timeout, or cancellation identity. The pipeline task + // faults first so a consumer woken by the ready source never races its retirement. + CompletePipelineTask(fault, runContinuationsAsynchronously: true); + FaultReady(fault); + return; } - // Static continuation: a bridge into framework state, so no captured scheduling context is needed. - waiter.OnCompleted(static state => + if (IsCancelRequested) + RequestBackendCancellation(); + if (!CompleteReady(null, runContinuationsAsynchronously: onExecutorStrand)) { - var flow = (CommandFlow)state!; - var ctx = flow._context; - // A faulted activation never claimed the shared promise; complete only this flow's source. - var activation = ctx.GetDecoderAsync().GetAwaiter(); - if (!activation.IsCompletedSuccessfully) - { - // Preserve the activation's close, timeout, or cancellation identity. - try { activation.GetResult(); } - catch (Exception ex) { flow._executePipelinedCore.SetException(ex); } - return; - } - var promise = ctx.GetProtocolStatic().Promise; - PromiseAsyncValueTaskMethodBuilder.Promise = promise; - ValueTask task = flow.ExecutePipelined(ctx); - try - { - if (!task.IsCompleted) - { - ((IValueTaskSource)promise).OnCompleted(static state => - { - var flow = (CommandFlow)state!; - try - { - var promise = flow._context - .GetProtocolStatic().Promise; - ((IValueTaskSource)promise).GetResult(promise.Token); - flow._executePipelinedCore.SetResult(true); - } - catch (Exception ex) - { - flow._executePipelinedCore.SetException(ex); - } - // This internal bridge runs no user code and requires no ExecutionContext flow. - }, flow, promise.Token, ValueTaskSourceOnCompletedFlags.None); - } - else - { - try - { - task.GetAwaiter().GetResult(); - flow._executePipelinedCore.SetResult(true); - } - catch (Exception ex) - { - flow._executePipelinedCore.SetException(ex); - } - } - } - finally - { - PromiseAsyncValueTaskMethodBuilder.Promise = null; - } - }, this); + // Teardown released the consumer while this flow waited for its turn. Nothing reads the + // response, the closing wire owns it. + CompletePipelineTask(null, runContinuationsAsynchronously: true); + return; + } + // A cancel latched before activation may have released its caller already. The response + // still has to reach RFQ, so drain it unless a consumer already owns the decoder. + if (IsCancelRequested) + TryTakeOverDrain(); + } - return new ValueTask(this, _executePipelinedCore.Version); + void FaultReady(Exception exception) + { + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + CompleteReady(exception, runContinuationsAsynchronously: true); } - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder))] - async ValueTask ExecutePipelined(Context context) + bool CompleteReady(Exception? exception, bool runContinuationsAsynchronously) + { + if (Interlocked.CompareExchange(ref _state.ReadyCompletion, 1, 0) != 0) + return false; + if (exception is null) + _state.ReadySource.SetResult(true, runContinuationsAsynchronously); + else + _state.ReadySource.SetException(exception, runContinuationsAsynchronously); + return true; + } + + void CompletePipelineTask(Exception? exception, bool runContinuationsAsynchronously = false) { - // Pre-start teardown may have already claimed terminality. In that case the consumer has its - // fault and this late dispatch has no body tenure to establish. - if (Interlocked.CompareExchange(ref _bodyState, BodyRunning, BodyNotStarted) != BodyNotStarted) + if (Interlocked.Exchange(ref _state.Phase, PhaseCompleted) is PhaseCompleted) return; - try + if (exception is null) + _state.PipelineTaskSource.SetResult(true, runContinuationsAsynchronously); + else + _state.PipelineTaskSource.SetException(exception, runContinuationsAsynchronously); + } + + void EnsureSyncHandoff() + { + if (_ops.IsAsyncAtDispatch) + ThrowHelper.ThrowInvalidOperation( + "Synchronous result consumption requires a flow initialized for synchronous execution."); + if (_state.SyncHandoffClaimed) + return; + _ops.WaitForSyncHandoff(); + _state.SyncHandoffClaimed = true; + } + + internal bool MoveNext() + { + EnsureSyncHandoff(); + while (true) { - // If we have a continuation stored we must already be on the caller thread, - // otherwise we must make sure to unblock the executor (see comment in the write phase). - // This first handoff is body execution too: close may fault it, so it belongs inside the - // same terminal envelope that publishes the consumer fault and body termination. - if (!IsAsync && !_callerInteractionCore.HasHandoff) - await YieldToCaller(); - - // User cancellation must not cancel activation or wire I/O. The flow observes it after - // activation, drains itself to RFQ, then delivers OCE without invoking pipeline recovery. - _decoder = await context.GetDecoderAuto().ConfigureAwait(false); - var publishedResult = false; - while (++_commandIndex < CommandCount) + var phase = Volatile.Read(ref _state.Phase); + switch (phase) { - _isResultReady = false; - bool hasPreparedDescription; - bool suppressEnumeration; - bool describeForPreparation; - { - ref readonly var command = ref _commands.ItemRef(_commandIndex); - _decoder.UseReadTimeout(command.Timeout); - suppressEnumeration = command.SuppressEnumeration; - describeForPreparation = command.DescribeForPreparation; - hasPreparedDescription = command.Descriptor is { IsPrepared: true, PreparedRowDescription: not null } - && !command.DescribeOnly; - } + case PhaseInitial: + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseInitial) != PhaseInitial) + continue; + _state.CommandIndex = 0; + return First(); + case PhaseResultReady: + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) != PhaseResultReady) + continue; + return NextBatch(); + case PhaseReading: + ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow."); + return false; + case PhaseDraining: + _ops.Flow.WaitForCompleteSynchronously(); + throw Volatile.Read(ref _state.ColdState)?.TerminalException + ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); + default: + if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } terminal) + ExceptionDispatchInfo.Throw(terminal); + return false; + } + } + } - // Registrations only latch and wake; terminal delivery remains body-owned. Dispose them - // before promise tenure ends so callbacks cannot reach the next flow. Do not rearm after - // consumer disposal, where a persistent cancellation could escape its intended wait. - if (Volatile.Read(ref _cancellationState) is { } cancellationAtReadStart && !IsDraining - && (cancellationAtReadStart.CallerToken.CanBeCanceled - || cancellationAtReadStart.FlowToken.CanBeCanceled)) - RegisterCancellationCallbacks(cancellationAtReadStart); - // After close, a fresh command must not consume bytes left by its predecessor. A draining - // flow may continue reading its own response to restore RFQ. - if (!IsDraining && context.IsProtocolClosed) - throw context.FlowTerminationException; - - PreparationReadState? preparationRead = null; - if (describeForPreparation) - { - preparationRead = new(); - await ReadPreparationDescription(context, preparationRead).ConfigureAwait(false); - } - else if (IsAsync && hasPreparedDescription) - { - // Prepared commands with a known description have the compact BindComplete -> - // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes - // this outer body rather than a nested parser coroutine; the second message normally - // comes from the same batch and is consumed synchronously. - if (!_decoder.TryMoveNext()) - { - if (!await _decoder.MoveNextAsync().ConfigureAwait(false)) - _decoder.ThrowUnexpectedEof(); - } - var message = _decoder.Current; - - if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) - { - _pgError = bindError; - _requestedRowDescription = null; - } - else - { - if (!_decoder.TryMoveNext()) - { - if (!await _decoder.MoveNextAsync().ConfigureAwait(false)) - _decoder.ThrowUnexpectedEof(); - } - message = _decoder.Current; - message.DebugEnsureExpected(PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); - _pgError = null; - _requestedRowDescription = null; - } - } - else if (IsAsync) - { - var rowDescription = context.GetProtocolStatic().RowDescription; - var read = _commands.ItemRef(_commandIndex).ReadUntilExecuteAsync(_decoder, rowDescription); - (_pgError, _requestedRowDescription) = await read.ConfigureAwait(false); - } - else - { - var rowDescription = context.GetProtocolStatic().RowDescription; - (_pgError, _requestedRowDescription) = _commands.ItemRef(_commandIndex) - .ReadUntilExecute(_decoder, rowDescription); - } + bool First() + { + try + { + WaitForReadySynchronously(); + Debug.Assert(!_state.ConsumerDetached); + RegisterCancellation(default); + var result = ReadNextPublishedResult(); + return result is not null && PublishSynchronousResult(result); + } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + } - // A draining consumer cannot observe CommandResult, so retain read errors for disposal. - var capturedThisCommand = false; - var readErrorIsOwnCancellation = _pgError is { } readError - && IsOwnCancellation(readError); - if (IsConsumingAutonomously && _pgError is { } readErrorToCapture - && !readErrorIsOwnCancellation) - { - (_drainErrors ??= new()).Add(PgErrorException.Create(readErrorToCapture)); - capturedThisCommand = true; - } + bool NextBatch() + { + try + { + RegisterCancellation(default); + var result = _state.Current!; + var completeError = CompleteCurrentResult(); + _state.CurrentPublished = false; + if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } consumerFault) + { + Interlocked.Exchange(ref _state.Phase, PhaseDraining); + NotifyDrainStarted(); + _state.ConsumerDetached = true; + Drain(); + ExceptionDispatchInfo.Throw(consumerFault); + } + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + SkipDiscardedCommands(); - // Await in-flight callbacks before releasing shared promise tenure. - var cancellation = Volatile.Read(ref _cancellationState); - if (cancellation is not null) - await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); - // Cancellation switches to autonomous drain; terminal delivery follows RFQ and tenure release. - var effectiveCancellationToken = GetEffectiveCancellationToken(cancellation); - if ((cancellation is { } && Volatile.Read(ref cancellation.Requested) - || effectiveCancellationToken.IsCancellationRequested) && !IsEnumerationCompleted) - { - cancellation ??= GetOrCreateCancellationState(); - if (effectiveCancellationToken.IsCancellationRequested) - cancellation.DeliverToken = effectiveCancellationToken; - cancellation.DeliverOce = true; - if (!IsDraining) - MarkBodyInitiatedDrain(); - } + _state.CommandIndex++; + var next = ReadNextPublishedResult(); + if (next is not null) + return PublishSynchronousResult(next); - var result = InitializeResult( - context, preparationRead); - ((CommandFlowObserver?)GetObserver(out var observerState)) - ?.OnCommandResult(this, result, observerState); + return false; + } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + } - // Disposal drains without another result handoff. Graceful close instead faults the - // attached consumer, then uses the same autonomous drain. Command errors remain results. - if (context.StoppingToken.IsCancellationRequested && !IsDraining - && !IsEnumerationCompleted) - { - // Latch the close (a consumer that Resets past this point self-delivers it), wake a - // parked consumer, then drain. - var close = context.FlowTerminationException; - _callerInteractionCore.SetCloseLatch(close); - CompleteEnumerationWithClose(close); - MarkBodyInitiatedDrain(); - } - if (!IsDraining && !IsConsumingNonQuery && !suppressEnumeration) - { - // Eager async execution must wait for the consumer to arm generation zero before - // publishing its first result. Synchronous execution already runs on that caller. - if (!publishedResult && IsAsync) - { - await _callerInteractionCore.WaitForCaller(this).ConfigureAwait(false); - // The first consumer can arrive after the response prelude was already read - // and its registrations were disposed. Its token was armed by MoveNextAsync; - // the result has now won that race, so retire the late registration before - // publishing the result. - if (Volatile.Read(ref _cancellationState) is { } lateCancellation) - await DisposeCancellationRegistrations(lateCancellation).ConfigureAwait(false); - EnterStoppingDrainIfNeeded(context); - } - - if (!IsDraining && !IsConsumingNonQuery) - { - _isResultReady = true; - publishedResult = true; - // Result continuations run asynchronously so the body can reach the next gate - // before user code asks for the next result. Buffered batches then advance inline - // from MoveNextAsync instead of suspending one Task state machine per result. - PublishEnumeratorResult(context, result); - - if (!IsDraining && !IsConsumingNonQuery) - { - if (IsAsync) - { - await _callerInteractionCore.WaitForCaller(this).ConfigureAwait(false); - EnterStoppingDrainIfNeeded(context); - } - else - await YieldToCaller(); - - /* The next MoveNext or MoveNextAsync call resumes here. */ - } - } - } - else if (!_drainModeEntered && IsAsyncAtDispatch && !IsAsync) - { - // An async I/O wake raced a synchronous disposer before the body reached its handoff. - if (WaitForDrainOnDispose) - { - // Hand the continuation to the disposer, which waits on the rendezvous rather than - // this task and can therefore drive the remaining drain without sync-over-async. - await YieldToCaller(); - } - else - { - // No disposer is waiting to drive; retain asynchronous background draining. - IsAsync = IsAsyncAtDispatch; - } - } - // Preserve the drive mode chosen on first drain entry. - _drainModeEntered = _drainModeEntered || IsDraining; - - // Disposing the message enumerator completes the command and, in drain mode, consumes its - // remaining rows. Re-read the current execution mode after every resumption. - (PgError Error, TransactionStatus TransactionStatus)? completeError; - // Consumption mode may change while the body is suspended; use the current value. - if (IsConsumingNonQuery || suppressEnumeration) - { - await CompleteInternalConsumptionAsync( - result, suppressEnumeration, capturedThisCommand).ConfigureAwait(false); - continue; - } - else if (IsAsync) - { - var resultEnumerator = context.GetProtocolStatic().ResultMessageEnumerator; - await resultEnumerator.DisposeAsync().ConfigureAwait(false); - completeError = resultEnumerator.CompleteError; - } - else - { - var resultEnumerator = context.GetProtocolStatic().ResultMessageEnumerator; - resultEnumerator.Dispose(); - completeError = resultEnumerator.CompleteError; - } + bool PublishSynchronousResult(CommandResult result) + { + _state.Current = result; + _state.CurrentPublished = true; + Interlocked.Exchange(ref _state.Phase, PhaseResultReady); + var context = _state.Context; + if (!IsClosed && context.StoppingToken.IsCancellationRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, + context.FlowTerminationException, null); + if (!IsCancelRequested && !IsClosed) + return true; + + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + Drain(); + else + _ops.Flow.WaitForCompleteSynchronously(); + throw Volatile.Read(ref _state.ColdState)?.TerminalException + ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); + } - if (result.Error is not null || completeError is not null) - await HandleCommandErrorsAsync( - result, suppressEnumeration, consumeInternally: false, - capturedThisCommand, completeError).ConfigureAwait(false); - } + void WaitForReadySynchronously() + { + var ready = new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version); + if (ready.IsCompleted) + _ = ready.GetAwaiter().GetResult(); + else + _ = ready.AsTask().GetAwaiter().GetResult(); + } - // The framework observes trailing write failure before releasing this flow. - if (_readFlowRfq) + internal ValueTask MoveNextAsync(CancellationToken cancellationToken) + { + if (!_ops.IsAsyncAtDispatch) + return ValueTask.FromException(ThrowHelper.ThrowInvalidOperation( + "Asynchronous result consumption requires a flow initialized for asynchronous execution.")); + while (true) + { + var phase = Volatile.Read(ref _state.Phase); + switch (phase) { - if (_decoder.TryMoveNext()) - { - var message = _decoder.Current; - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - else if (IsAsync) - { - await ReadRfqAsync(_decoder).ConfigureAwait(false); - } - else - { - ReadRfq(_decoder); - } + case PhaseInitial: + if (cancellationToken.IsCancellationRequested) + return CancelBeforeRead(cancellationToken); + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseInitial) != PhaseInitial) + continue; + _state.CommandIndex = 0; + return FirstAsync(cancellationToken); + case PhaseResultReady: + if (cancellationToken.IsCancellationRequested) + return CancelBeforeRead(cancellationToken); + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) != PhaseResultReady) + continue; + return NextBatchAsync(cancellationToken); + case PhaseReading: + return ValueTask.FromException( + ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow.")); + case PhaseDraining: + return AwaitTakeoverAsync(); + default: + return Volatile.Read(ref _state.ColdState)?.TerminalException is { } terminal + ? ValueTask.FromException(terminal) + : new(false); } - - PublishEnumeratorResult(context, null); } - catch (PgClientClosedException) when (context.IsProtocolClosed) + } + + // A pre-cancelled token releases the caller immediately. The wire still drains to RFQ. + ValueTask CancelBeforeRead(CancellationToken cancellationToken) + { + RequestCancel(cancellationToken, CommandExecutionCancellationScope.CurrentWindow); + return ValueTask.FromException(new OperationCanceledException(cancellationToken)); + } + + // The consumer parks behind a takeover drain and receives the outcome that caused it. + async ValueTask AwaitTakeoverAsync() + { + await WaitForCompletionAsync().ConfigureAwait(false); + throw Volatile.Read(ref _state.ColdState)?.TerminalException ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask FirstAsync(CancellationToken cancellationToken) + { + Exception? deliver; + try { - // Scope to our own closure so a nested protocol's close doesn't get treated as ours. - // Latch the close so a consumer that Resets after this point self-delivers it. - _callerInteractionCore.SetCloseLatch(context.FlowTerminationException); - // A detached consumer treats close as drain completion; a live consumer observes the close. - if (IsDraining) + await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); + Debug.Assert(!_state.ConsumerDetached); + RegisterCancellation(cancellationToken); + var result = await ReadNextPublishedResultAsync().ConfigureAwait(false); + if (result is null) + return false; + _state.Current = result; + _state.CurrentPublished = true; + // Publish the idle state, then recheck the latches. A latch that landed between the read + // and this publication found no idle owner to take over, so this frame must act on it. + Interlocked.Exchange(ref _state.Phase, PhaseResultReady); + // Graceful stopping faults a result that arrives after the close began, as the ordinary + // flow does at each result boundary. Latch it so the drain delivers that close. + var context = _state.Context; + if (!IsClosed && context.StoppingToken.IsCancellationRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, + context.FlowTerminationException, null); + if (!IsCancelRequested && !IsClosed) + return true; + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) { - if (!IsEnumerationCompleted) - PublishEnumeratorResult(context, null); - return; + await DrainAsync().ConfigureAwait(false); + } + else + { + // The latching side took the decoder first. Park behind its drain. + await WaitForCompletionAsync().ConfigureAwait(false); } - CompleteEnumerationWithException(context.FlowTerminationException); + deliver = Volatile.Read(ref _state.ColdState)?.TerminalException; + } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); throw; } - catch (OperationCanceledException ex) when (IsCancellationToken(ex.CancellationToken)) + catch (Exception ex) { - CompleteEnumerationWithException(ex); + FaultFromOwner(ex); throw; } - catch (TimeoutException ex) + throw deliver ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask NextBatchAsync(CancellationToken cancellationToken) + { + try { - await HandleTimeoutAsync(context, ex).ConfigureAwait(false); - return; + RegisterCancellation(cancellationToken); + var result = _state.Current!; + var resultEnumerator = _state.Context.GetProtocolStatic() + .ResultMessageEnumerator; + await resultEnumerator.DisposeAsync().ConfigureAwait(false); + var completeError = resultEnumerator.CompleteError; + _state.CurrentPublished = false; + if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } consumerFault) + { + Interlocked.Exchange(ref _state.Phase, PhaseDraining); + NotifyDrainStarted(); + _state.ConsumerDetached = true; + await DrainAsync().ConfigureAwait(false); + ExceptionDispatchInfo.Throw(consumerFault); + } + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + await SkipDiscardedCommandsAsync().ConfigureAwait(false); + + _state.CommandIndex++; + var next = await ReadNextPublishedResultAsync().ConfigureAwait(false); + if (next is not null) + { + result = next; + _state.Current = result; + _state.CurrentPublished = true; + Interlocked.Exchange(ref _state.Phase, PhaseResultReady); + var context = _state.Context; + if (!IsClosed && context.StoppingToken.IsCancellationRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, + context.FlowTerminationException, null); + if (!IsCancelRequested && !IsClosed) + return true; + + if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) == PhaseResultReady) + await DrainAsync().ConfigureAwait(false); + else + await WaitForCompletionAsync().ConfigureAwait(false); + throw Volatile.Read(ref _state.ColdState)?.TerminalException + ?? ThrowHelper.ThrowUnexpected("A latched flow completed without a terminal outcome."); + } + + return false; } - catch (Exception ex) + catch (TimeoutException ex) { - CompleteEnumerationWithException(ex); + HandleReadTimeout(ex); throw; } - finally + catch (Exception ex) { - // The body is the sole owner of protocol-static row metadata. Recovery consumes only - // decoder/wire state, so a faulted body can release oversized storage while recovery - // retains the failed flow's framework tenure. - ref readonly var readState = ref context.GetProtocolStatic(); - readState.Reset(); - PublishBodyTerminated(); + FaultFromOwner(ex); + throw; } } - void PublishEnumeratorResult(Context context, CommandResult? next) + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask ReadNextPublishedResultAsync() { - var completed = next is null; - var publishAsync = IsAsync; - if (completed) + CommandResult? result = _state.Current; + while (_state.CommandIndex < _state.Commands.Count) { - _enumeratorCurrent = null; + result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); + _state.Current = result; + _state.CurrentPublished = false; + if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) + return result; + + var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); + var suppressedError = result.Error; + if (suppressedError is null && completeError is { } completionError) + suppressedError = completionError.Error; + if (suppressedError is not null) + { + var exception = PgErrorException.Create(suppressedError); + var cold = GetOrCreateColdState(); + Interlocked.CompareExchange(ref cold.TerminalException, exception, null); + (cold.DrainErrors ??= new()).Add(exception); + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + await SkipDiscardedCommandsAsync().ConfigureAwait(false); + _state.CommandIndex++; + _state.Current = null; + Interlocked.Exchange(ref _state.Phase, PhaseDraining); + NotifyDrainStarted(); + _state.ConsumerDetached = true; + await DrainAsync().ConfigureAwait(false); + throw exception; + } + + _state.CommandIndex++; } - else + + if (result is null) + throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); + await CompleteBatchAsync().ConfigureAwait(false); + _state.ConsumerObservedCompletion = true; + return null; + } + + CommandResult? ReadNextPublishedResult() + { + CommandResult? result = _state.Current; + while (_state.CommandIndex < _state.Commands.Count) { - if (Volatile.Read(ref _cancellationState) is { } cancellation) - cancellation.CallerToken = default; + result = ReadResult(_state.CommandIndex); + _state.Current = result; + _state.CurrentPublished = false; + if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) + return result; + + var completeError = CompleteCurrentResult(); + var suppressedError = result.Error; + if (suppressedError is null && completeError is { } completionError) + suppressedError = completionError.Error; + if (suppressedError is not null) + { + var exception = PgErrorException.Create(suppressedError); + var cold = GetOrCreateColdState(); + Interlocked.CompareExchange(ref cold.TerminalException, exception, null); + (cold.DrainErrors ??= new()).Add(exception); + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + SkipDiscardedCommands(); + _state.CommandIndex++; + _state.Current = null; + Interlocked.Exchange(ref _state.Phase, PhaseDraining); + NotifyDrainStarted(); + _state.ConsumerDetached = true; + Drain(); + throw exception; + } - if (!ReferenceEquals(_enumeratorCurrent, next)) - _enumeratorCurrent = next; + _state.CommandIndex++; } - // Close is durable across generations; complete the current one without publishing a result. - if (_callerInteractionCore.CloseException is not null) + if (result is null) + throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); + CompleteBatch(); + _state.ConsumerObservedCompletion = true; + return null; + } + + // Reads through the command's execute prelude and initializes the protocol-static result. + async ValueTask ReadResultAsync(int commandIndex) + { + var context = _state.Context; + var decoder = context.Decoder; + // After close, a fresh command must not consume bytes left by its predecessor. + if (context.IsProtocolClosed) + throw context.FlowTerminationException; + PgError? error; + RowDescription? requestedRowDescription; + ref readonly var command = ref _state.Commands.ItemRef(commandIndex); + var describeOnly = command.DescribeOnly; + var hasPreparedDescription = command.Descriptor + is { IsPrepared: true, PreparedRowDescription: not null }; + decoder.UseReadTimeout(command.Timeout); + ParameterTypeList? preparationParameterTypes = null; + if (command.DescribeForPreparation) { - CompleteEnumerationWithClose(_callerInteractionCore.CloseException); - return; + var preparation = await command.ReadPreparationDescriptionAsync( + decoder, context.GetProtocolStatic().RowDescription) + .ConfigureAwait(false); + error = preparation.Item1; + preparationParameterTypes = preparation.Item2; + requestedRowDescription = preparation.Item3; } - if (completed) + else if (hasPreparedDescription && !describeOnly) { - // Publish durable terminal state atomically with respect to consumer rearming. Async - // consumers complete from the protocol scheduler so they cannot reenter this lock or - // the pipeline frame that still owns the shared promise; sync consumers retain their - // caller-driven completion. - using (_rearmLock.EnterScope()) + // Prepared commands with a known description have the compact BindComplete -> + // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes this + // frame rather than a nested parser coroutine. + if (!decoder.TryMoveNext()) { - PublishEnumerationCompleted(); - if (!publishAsync) - CompleteEnumeration(); + if (!await decoder.MoveNextAsync().ConfigureAwait(false)) + decoder.ThrowUnexpectedEof(); } - if (publishAsync) - SubmitPublication(context, DetachedPublication.Completion); - return; + var message = decoder.Current; + if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) + { + error = bindError; + } + else + { + if (!decoder.TryMoveNext()) + { + if (!await decoder.MoveNextAsync().ConfigureAwait(false)) + decoder.ThrowUnexpectedEof(); + } + decoder.Current.DebugEnsureExpected(PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); + error = null; + } + requestedRowDescription = null; } - if (publishAsync) + else { - // Queue the publication itself so the body reaches its next caller gate before user code - // resumes. Routing through the protocol scheduler preserves that ordering without forcing - // every result continuation onto the ThreadPool. - SubmitPublication(context, DetachedPublication.Result); + (error, requestedRowDescription) = await command + .ReadUntilExecuteAsync(decoder, context.GetProtocolStatic().RowDescription) + .ConfigureAwait(false); } - else - TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); + return InitializeResult( + commandIndex, error, requestedRowDescription, preparationParameterTypes); } - async ValueTask HandleTimeoutAsync(Context context, TimeoutException exception) + CommandResult ReadResult(int commandIndex) { - CompleteEnumerationWithException(exception); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, - BackendCancellationTiming.AtReadFrontier, allowCompletedEnumeration: true); + var context = _state.Context; + var decoder = context.Decoder; if (context.IsProtocolClosed) - ExceptionDispatchInfo.Throw(exception); - - // The timeout is terminal for the consumer, not for the body. Keep ownership of the - // command sequence and drain every remaining RFQ window; reaching each RFQ requests - // cancellation for each following window through the cancellation coordinator. Recovery is - // reserved for a failure of this semantic drain, where only wire obligations remain. - if (Volatile.Read(ref _cancellationState) is { } cancellation) - await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); - ((CommandFlowObserver?)GetObserver(out var observerState)) - ?.OnDrainStarted(this, observerState); - try + throw context.FlowTerminationException; + ref readonly var command = ref _state.Commands.ItemRef(commandIndex); + decoder.UseReadTimeout(command.Timeout); + PgError? error; + RowDescription? requestedRowDescription; + ParameterTypeList? preparationParameterTypes = null; + if (command.DescribeForPreparation) { - while (context.OutstandingRfqCount != 0) - _ = await _decoder!.GetNextAuto().ConfigureAwait(false); + var preparation = command.ReadPreparationDescription( + decoder, context.GetProtocolStatic().RowDescription); + error = preparation.Item1; + preparationParameterTypes = preparation.Item2; + requestedRowDescription = preparation.Item3; } - catch (TimeoutException) + else { - // The semantic drain owns the same cancellation episode. A timeout here would otherwise - // leave the episode unaware that its first read-timeout escalation made no protocol progress. - RequestCancel(default, CancellationScope.RemainingFlow, - BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier, - allowCompletedEnumeration: true); - throw; + (error, requestedRowDescription) = command.ReadUntilExecute( + decoder, context.GetProtocolStatic().RowDescription); } + return InitializeResult( + commandIndex, error, requestedRowDescription, preparationParameterTypes); } - static async ValueTask ReadRfqAsync(PgDecoder decoder) - { - var message = await decoder.GetNextAsync().ConfigureAwait(false); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - - static void ReadRfq(PgDecoder decoder) - { - var message = decoder.GetNext(); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - - [MethodImpl(MethodImplOptions.NoInlining)] CommandResult InitializeResult( - Context context, PreparationReadState? preparationRead) + int commandIndex, PgError? error, RowDescription? requestedRowDescription, + ParameterTypeList? preparationParameterTypes = null) { - ref readonly var readState = ref context.GetProtocolStatic(); - readState.ResultMessageEnumerator.Initialize(_commands.ItemRef(_commandIndex), _decoder!); - var result = _enumeratorCurrent ?? readState.CommandResult; - - ref readonly var command = ref _commands.ItemRef(_commandIndex); + var context = _state.Context; + ref readonly var readState = ref context.GetProtocolStatic(); + ref readonly var command = ref _state.Commands.ItemRef(commandIndex); + readState.ResultMessageEnumerator.Initialize(command, context.Decoder); + var result = readState.CommandResult; var descriptor = command.Descriptor; - // We were preparing and we have no error from parse, make a prepared descriptor. + // A named unprepared statement that parsed becomes a prepared descriptor. if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault - && (_pgError is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) + && (error is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) { - descriptor = CommandDescriptor.CreatePrepared( - descriptor.CommandName, - preparationRead?.ParameterTypes ?? descriptor.ParameterTypes, - _requestedRowDescription?.Preserve()); + descriptor = CommandDescriptor.CreatePrepared(descriptor.CommandName, + preparationParameterTypes ?? descriptor.ParameterTypes, + requestedRowDescription?.Preserve()); } - result.Initialize(this, _commandIndex, descriptor, _requestedRowDescription, - !command.DescribeOnly, command.IsSimple(), _pgError); + result.Initialize(_ops.Flow, commandIndex, descriptor, requestedRowDescription, + !command.DescribeOnly, command.IsSimple(), error); + _ops.OnCommandResult(result); return result; } - ValueTask ReadPreparationDescription(Context context, PreparationReadState state) + async ValueTask<(PgError Error, TransactionStatus TransactionStatus)?> CompleteCurrentResultAsync() { - var rowDescription = context.GetProtocolStatic().RowDescription; - ref readonly var command = ref _commands.ItemRef(_commandIndex); - if (!IsAsync) - { - (_pgError, state.ParameterTypes, _requestedRowDescription) = - command.ReadPreparationDescription(_decoder!, rowDescription); - return default; - } - - var read = command.ReadPreparationDescriptionAsync(_decoder!, rowDescription); - if (!read.IsCompletedSuccessfully) - return AwaitRead(this, state, read); - (_pgError, state.ParameterTypes, _requestedRowDescription) = read.Result; - return default; + var enumerator = _state.Context.GetProtocolStatic().ResultMessageEnumerator; + await enumerator.DisposeAsync().ConfigureAwait(false); + return enumerator.CompleteError; + } - static async ValueTask AwaitRead( - CommandFlow flow, PreparationReadState state, - ValueTask<(PgError?, ParameterTypeList, RowDescription?)> read) - { - (flow._pgError, state.ParameterTypes, flow._requestedRowDescription) = - await read.ConfigureAwait(false); - } + (PgError Error, TransactionStatus TransactionStatus)? CompleteCurrentResult() + { + var enumerator = _state.Context.GetProtocolStatic().ResultMessageEnumerator; + enumerator.Dispose(); + return enumerator.CompleteError; } - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] - async ValueTask CompleteInternalConsumptionAsync( - CommandResult result, bool suppressEnumeration, bool capturedThisCommand) + async ValueTask SkipDiscardedCommandsAsync() { - while (_decoder!.Current.Header.Type is PgTypes.BackendType.DataRow) - { - if (!_decoder.TryMoveNext()) - await _decoder.GetNextAsync().ConfigureAwait(false); - } - result.CompleteNonQuery(_decoder.Current); - var completeError = await _commands.ItemRef(_commandIndex) - .CompleteAsync(_decoder).ConfigureAwait(false); - if (_pgError is null && completeError is null) - { - var recordsAffected = result.GetCommandComplete().BatchRecordsAffected; - if (recordsAffected >= 0) - _nonQueryRecordsAffected = _nonQueryRecordsAffected < 0 - ? recordsAffected - : checked(_nonQueryRecordsAffected + recordsAffected); - } + while (++_state.CommandIndex < _state.Commands.Count && !_state.Commands[_state.CommandIndex].WithSync) { } + await ReadRfqAsync().ConfigureAwait(false); + if (_state.CommandIndex == _state.Commands.Count) + _state.ReadFlowRfq = false; + } - if (result.Error is not null || completeError is not null) - await HandleCommandErrorsAsync( - result, suppressEnumeration, consumeInternally: true, - capturedThisCommand, completeError).ConfigureAwait(false); + void SkipDiscardedCommands() + { + while (++_state.CommandIndex < _state.Commands.Count && !_state.Commands[_state.CommandIndex].WithSync) { } + ReadRfq(); + if (_state.CommandIndex == _state.Commands.Count) + _state.ReadFlowRfq = false; } - async ValueTask HandleCommandErrorsAsync( - CommandResult result, bool suppressEnumeration, bool consumeInternally, - bool capturedThisCommand, - (PgError Error, TransactionStatus TransactionStatus)? completeError) + async ValueTask ReadRfqAsync() { - var resultErrorIsOwnCancellation = result.Error is { } resultError - && IsOwnCancellation(resultError); - if (suppressEnumeration && result.Error is { } suppressedError - && !resultErrorIsOwnCancellation) - { - (_drainErrors ??= new()).Add(PgErrorException.Create(suppressedError)); - capturedThisCommand = true; - if (!IsDraining) - MarkBodyInitiatedDrain(); - } + var message = await _state.Context.Decoder.GetNextAsync().ConfigureAwait(false); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } - // Accumulate each command's fresh error while draining, but do not duplicate an error - // already captured during its read phase or delivered to a live consumer. - var completeErrorIsOwnCancellation = completeError is { } completedWithError - && IsOwnCancellation(completedWithError.Error); - if ((consumeInternally || IsConsumingNonQuery || IsDraining && !_isResultReady) - && !capturedThisCommand && completeError is { } error - && !completeErrorIsOwnCancellation) - (_drainErrors ??= new()).Add(PgErrorException.Create(error.Error)); - - // Extended-query errors discard every following command through the next Sync. Skip - // those commands locally and consume the RFQ which is their only wire response. - if (completeError is not { TransactionStatus: TransactionStatus.Unknown }) - return; + void ReadRfq() + { + var message = _state.Context.Decoder.GetNext(); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } - while (++_commandIndex < CommandCount && !_commands[_commandIndex].WithSync) { } + async ValueTask CompleteBatchAsync() + { + if (_state.ReadFlowRfq) + await ReadRfqAsync().ConfigureAwait(false); + await DisposeRegistrationsAsync().ConfigureAwait(false); + Finish(); + } - if (IsAsync) - await ReadRfqAsync(_decoder!).ConfigureAwait(false); - else - ReadRfq(_decoder!); + void CompleteBatch() + { + if (_state.ReadFlowRfq) + ReadRfq(); + DisposeRegistrations(); + Finish(); + } - // Reaching the end means the discarded segment terminated at our appended Sync. - if (_commandIndex == CommandCount) - _readFlowRfq = false; + // The wire is at this command's RFQ. Release the shared read objects, record the outcome the + // consumer must observe, then complete the pipeline task. + void Finish() + { + _state.Context.GetProtocolStatic().Reset(); + _state.Current = null; + _state.CurrentPublished = false; + if (IsCancelRequested) + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, + new OperationCanceledException(_state.ColdState!.DeliverToken), null); + else if (Volatile.Read(ref _state.ColdState)?.CloseException is { } close) + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, close, null); + CompletePipelineTask(null); } - void SubmitPublication(Context context, DetachedPublication publication) + bool IsOwnCancellation(PgError error) + => IsCancelRequested && error.SqlState == PgErrorCodes.QueryCanceled; + + // The owning frame failed. The read error is the pipeline task's failure, which the framework + // recovers or drains. Later consumer calls replay it. + void FaultFromOwner(Exception exception) { - Debug.Assert(_detachedPublication is DetachedPublication.None); - _detachedPublication = publication; - context.SubmitDetached((IThreadPoolWorkItem)this); + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + if (Volatile.Read(ref _state.Phase) == PhaseCompleted) + return; + DisposeRegistrations(); + _state.Current = null; + _state.CurrentPublished = false; + if (HasDecoder) + _state.Context.GetProtocolStatic().Reset(); + CompletePipelineTask(exception); } - private protected override void ExecuteDetachedWorkItem() + // Takes an idle decoder for a drain. Returns false when a frame already owns it, which will observe + // the latch itself, or when the flow reached its terminal. + bool TryTakeOverDrain() { - var publication = _detachedPublication; - _detachedPublication = DetachedPublication.None; - switch (publication) + while (true) { - case DetachedPublication.Result: - TrySetEnumeratorResult(true, runContinuationsAsynchronously: false); - break; - case DetachedPublication.Completion: - CompleteEnumeration(runContinuationsAsynchronously: false); - break; - default: - ThrowHelper.ThrowInvalidOperation("The command flow has no publication pending."); - break; + var phase = Volatile.Read(ref _state.Phase); + if (phase is PhaseInitial && !HasDecoder) + return false; + if (phase is not (PhaseInitial or PhaseResultReady)) + return false; + if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) + continue; + NotifyDrainStarted(); + _state.ConsumerDetached = true; + ThreadPool.UnsafeQueueUserWorkItem(static state => + _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ops.Flow); + return true; } } - void SetCallerCancellationToken(CancellationToken token) + void NotifyDrainStarted() { - var cancellation = GetOrCreateCancellationState(); - lock (cancellation) + if (Interlocked.Exchange(ref _state.DrainStarted, 1) is 0) + _ops.OnDrainStarted(); + } + + // Autonomous drain. Owns the decoder until the pipeline task completes. Never throws. + async ValueTask DrainAsync() + { + try + { + var result = _state.Current; + if (result is null) + { + await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); + if (_state.CommandIndex < 0) + _state.CommandIndex = 0; + if (_state.CommandIndex >= _state.Commands.Count) + { + await CompleteBatchAsync().ConfigureAwait(false); + return; + } + result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); + } + + while (true) + { + var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); + CaptureDrainError(result, completeError); + _state.CurrentPublished = false; + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + await SkipDiscardedCommandsAsync().ConfigureAwait(false); + if (++_state.CommandIndex >= _state.Commands.Count) + break; + result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); + } + await CompleteBatchAsync().ConfigureAwait(false); + } + catch (TimeoutException ex) { - cancellation.CallerToken = token; - RegisterCancellationCallbacksLocked(cancellation); + // A timeout during semantic drain escalates the same cancellation episode immediately. + // The pipeline failure then hands any remaining wire obligation to recovery. + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, + BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); + FaultFromOwner(ex); + } + catch (Exception ex) + { + FaultFromOwner(ex); } } - void RegisterCancellationCallbacks(CancellationState cancellation) + // A read timeout is terminal for the consumer but not for the wire obligation. This frame has + // released the decoder read tenure, so transfer ownership to an autonomous semantic drain while + // the original MoveNext returns its timeout immediately. + void HandleReadTimeout(TimeoutException exception) { - lock (cancellation) - RegisterCancellationCallbacksLocked(cancellation); + Interlocked.CompareExchange( + ref GetOrCreateColdState().TerminalException, exception, null); + _state.ConsumerDetached = true; + NotifyDrainStarted(); + Interlocked.Exchange(ref _state.Phase, PhaseDraining); + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, + BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); + ThreadPool.UnsafeQueueUserWorkItem(static state => + _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ops.Flow); } - void RegisterCancellationCallbacksLocked(CancellationState cancellation) + void Drain() { - if (cancellation.CallerToken.CanBeCanceled) + try { - Debug.Assert(IsAsync); - if (cancellation.CallerRegistration == default) - cancellation.CallerRegistration = cancellation.CallerToken.UnsafeRegister(static (state, token) - => ((CommandFlow)state!).RequestCancelAndWake(token, CancellationScope.CurrentWindow), this); + var result = _state.Current; + if (result is null) + { + // Synchronous disposal before any read. The activation bridge normally completed long + // ago, so this bridge is rarely more than a status check. + new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).AsTask().GetAwaiter().GetResult(); + if (_state.CommandIndex < 0) + _state.CommandIndex = 0; + if (_state.CommandIndex >= _state.Commands.Count) + { + CompleteBatch(); + return; + } + result = ReadResult(_state.CommandIndex); + } + + while (true) + { + var completeError = CompleteCurrentResult(); + CaptureDrainError(result, completeError); + _state.CurrentPublished = false; + if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + SkipDiscardedCommands(); + if (++_state.CommandIndex >= _state.Commands.Count) + break; + result = ReadResult(_state.CommandIndex); + } + CompleteBatch(); } - if (cancellation.FlowToken.CanBeCanceled && cancellation.FlowRegistration == default) + catch (Exception ex) { - cancellation.FlowRegistration = cancellation.FlowToken.UnsafeRegister(static (state, token) - => ((CommandFlow)state!).RequestCancelAndWake(token, CancellationScope.RemainingFlow), this); + FaultFromOwner(ex); } } - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] - async ValueTask DisposeCancellationRegistrations(CancellationState cancellation) + internal ValueTask DisposeAsync() { - CancellationTokenRegistration callerRegistration; - CancellationTokenRegistration flowRegistration; - lock (cancellation) + while (true) { - callerRegistration = cancellation.CallerRegistration; - cancellation.CallerRegistration = default; - flowRegistration = cancellation.FlowRegistration; - cancellation.FlowRegistration = default; + var phase = Volatile.Read(ref _state.Phase); + switch (phase) + { + case PhaseInitial: + case PhaseResultReady: + if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) + continue; + NotifyDrainStarted(); + _state.ConsumerDetached = true; + if (_state.Current is { IsComplete: false } + || _state.CommandIndex + 1 < _state.Commands.Count) + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + return _state.WaitForDrainOnDispose ? DisposeDrainAsync() : FireAndForgetDrain(); + case PhaseReading: + _state.ConsumerDetached = true; + NotifyDrainStarted(); + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + return _state.WaitForDrainOnDispose ? DisposeCompletedAsync() : default; + default: + return !_state.WaitForDrainOnDispose || _state.ConsumerObservedCompletion + ? default + : DisposeCompletedAsync(); + } } - await callerRegistration.DisposeAsync().ConfigureAwait(false); - await flowRegistration.DisposeAsync().ConfigureAwait(false); } - bool IsCancellationToken(CancellationToken token) + // Drains on the disposer's frame, then waits for framework release so a drain error can surface. + async ValueTask DisposeDrainAsync() { - var cancellation = Volatile.Read(ref _cancellationState); - return cancellation is not null - && (token == cancellation.CallerToken || token == cancellation.FlowToken); + await DrainAsync().ConfigureAwait(false); + await DisposeCompletedAsync().ConfigureAwait(false); } - // Cancellation callbacks only latch intent and wake the body. The body delivers cancellation after - // it has restored the wire boundary and is ready to release execution tenure. - internal Task CancelAsync() + ValueTask FireAndForgetDrain() { - var delivery = GetOrCreateCancelDelivery(); - if (IsCompleted) - { - delivery.TrySetResult(); - return delivery.Task; - } - RequestCancelAndWake(default, CancellationScope.RemainingFlow); - if (IsCompleted) - delivery.TrySetResult(); - return delivery.Task; + _ = DrainAsync(); + return default; } - CancellationState GetOrCreateCancellationState() + async ValueTask DisposeCompletedAsync() { - if (Volatile.Read(ref _cancellationState) is { } cancellation) - return cancellation; - var created = new CancellationState(); - return Interlocked.CompareExchange(ref _cancellationState, created, null) ?? created; + await WaitForCompletionAsync().ConfigureAwait(false); + ThrowDrainErrors(); } - TaskCompletionSource GetOrCreateCancelDelivery() + // Flow completion is independent of errors accumulated while draining. A close is a clean + // terminal for a disposing consumer. + async ValueTask WaitForCompletionAsync() { - var cancellation = GetOrCreateCancellationState(); - var delivery = Volatile.Read(ref cancellation.Delivery); - if (delivery is not null) - return delivery; - var created = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - return Interlocked.CompareExchange(ref cancellation.Delivery, created, null) ?? created; + try + { + await _ops.Flow.WaitForComplete().ConfigureAwait(false); + } + catch (PgClientClosedException) + { + } } - bool RequestCancel(CancellationToken token, CancellationScope scope, - BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, - BackendCancellationTiming subsequentTiming = BackendCancellationTiming.AfterGrace, - bool allowCompletedEnumeration = false) + internal void Dispose() { - if (IsEnumerationCompleted && !allowCompletedEnumeration) - return false; - var cancellation = GetOrCreateCancellationState(); - cancellation.DeliverToken = token; - var observedScope = Volatile.Read(ref cancellation.Scope); - while ((int)scope > observedScope) + if (!_ops.IsAsyncAtDispatch) + EnsureSyncHandoff(); + while (true) { - var priorScope = Interlocked.CompareExchange(ref cancellation.Scope, (int)scope, observedScope); - if (priorScope == observedScope) - break; - observedScope = priorScope; + var phase = Volatile.Read(ref _state.Phase); + switch (phase) + { + case PhaseInitial: + case PhaseResultReady: + if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) + continue; + NotifyDrainStarted(); + _state.ConsumerDetached = true; + if (_state.Current is { IsComplete: false } + || _state.CommandIndex + 1 < _state.Commands.Count) + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + Drain(); + if (_state.WaitForDrainOnDispose) + DisposeCompleted(); + return; + case PhaseReading: + _state.ConsumerDetached = true; + NotifyDrainStarted(); + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + if (_state.WaitForDrainOnDispose) + DisposeCompleted(); + return; + default: + if (_state.WaitForDrainOnDispose && !_state.ConsumerObservedCompletion) + DisposeCompleted(); + return; + } } - Volatile.Write(ref cancellation.Requested, true); - Volatile.Write(ref _draining, true); - var observedTiming = Volatile.Read(ref cancellation.Timing); - while ((int)timing > observedTiming) + } + + void DisposeCompleted() + { + try { - var priorTiming = Interlocked.CompareExchange(ref cancellation.Timing, (int)timing, observedTiming); - if (priorTiming == observedTiming) - break; - observedTiming = priorTiming; + _ops.Flow.WaitForCompleteSynchronously(); } - var observedSubsequentTiming = Volatile.Read(ref cancellation.SubsequentTiming); - while ((int)subsequentTiming > observedSubsequentTiming) + catch (PgClientClosedException) { - var priorTiming = Interlocked.CompareExchange(ref cancellation.SubsequentTiming, - (int)subsequentTiming, observedSubsequentTiming); - if (priorTiming == observedSubsequentTiming) - break; - observedSubsequentTiming = priorTiming; } - var delivery = Volatile.Read(ref cancellation.Delivery); - RequestBackendCancellation(timing, delivery); - return true; + ThrowDrainErrors(); } - void RequestCancelAndWake(CancellationToken token, CancellationScope scope) + void CaptureDrainError(CommandResult result, + (PgError Error, TransactionStatus TransactionStatus)? completeError) { - if (!RequestCancel(token, scope)) + if (!_state.ConsumerDetached || _state.CurrentPublished) return; - var delivery = Volatile.Read(ref _cancellationState) is { } cancellation - ? Volatile.Read(ref cancellation.Delivery) - : null; - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); - _callerInteractionCore.WakeBody(useDedicatedDriver: !IsAsync && delivery is not null); + var error = result.Error ?? completeError?.Error; + if (error is null || IsOwnCancellation(error)) + return; + var cold = GetOrCreateColdState(); + (cold.DrainErrors ??= new()).Add(PgErrorException.Create(error)); } - void RequestBackendCancellation(BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, - TaskCompletionSource? delivery = null) + void ThrowDrainErrors() { - // Cancellation which wins before body entry remains latched in the cancellation state and is - // replayed immediately after the execution context is published. - if (Volatile.Read(ref _contextPublished) && Volatile.Read(ref _cancellationState) is { } cancellation) + if (Volatile.Read(ref _state.ColdState)?.DrainErrors is not { Count: > 0 } errors) + return; + if (errors.Count is 1) + ExceptionDispatchInfo.Throw(errors[0]); + throw new AggregateException(errors); + } + + // When true, disposal waits for the drain to reach RFQ and for framework release. Otherwise it + // returns while the drain continues autonomously. + void RegisterCancellation(CancellationToken callerToken) + { + // Keep the second token/registration pair off ordinary flow objects. Default-token traffic + // does not need cancellation state at all and remains the allocation/footprint hot path. + var cancellation = Volatile.Read(ref _state.ColdState); + if (callerToken.CanBeCanceled || cancellation is not null) { - var key = Volatile.Read(ref cancellation.EpisodeKey); - if (key is null) + cancellation ??= GetOrCreateColdState(); + if (callerToken != cancellation.CallerToken) { - var created = new object(); - key = Interlocked.CompareExchange(ref cancellation.EpisodeKey, created, null) ?? created; + var callerRegistration = cancellation.CallerRegistration; + cancellation.CallerRegistration = default; + callerRegistration.Dispose(); + cancellation.CallerToken = callerToken; } - _context.RequestBackendCancellation(this, CancellationWindow, timing, delivery, - key, Volatile.Read(ref cancellation.Scope), - (BackendCancellationTiming)Volatile.Read(ref cancellation.SubsequentTiming)); + if (callerToken.CanBeCanceled && cancellation.CallerRegistration == default) + cancellation.CallerRegistration = callerToken.UnsafeRegister(static (state, token) + => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( + token, CommandExecutionCancellationScope.CurrentWindow), _ops.Flow); } - } - bool IsOwnCancellation(PgError error) - { - if (Volatile.Read(ref _cancellationState) is not { } cancellation - || !Volatile.Read(ref cancellation.Requested) || error.SqlState != PgErrorCodes.QueryCanceled) - return false; - // PgDecoder records each ErrorResponse arrival exactly once. Classification may inspect the - // preserved error through several command-result paths, so it must not report the same strike. - return true; + if (_state.FlowToken.CanBeCanceled && _state.FlowRegistration == default) + _state.FlowRegistration = _state.FlowToken.UnsafeRegister(static (state, token) + => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( + token, CommandExecutionCancellationScope.RemainingFlow), _ops.Flow); } - void EnterStoppingDrainIfNeeded(Context context) + ValueTask DisposeRegistrationsAsync() { - if (_callerInteractionCore.CloseException is { } close && context.StoppingToken.IsCancellationRequested - && !IsDraining && !IsEnumerationCompleted) + var cancellation = Volatile.Read(ref _state.ColdState); + var callerRegistration = cancellation?.CallerRegistration ?? default; + if (callerRegistration == default && _state.FlowRegistration == default) + return default; + if (cancellation is not null) + cancellation.CallerRegistration = default; + var flowRegistration = _state.FlowRegistration; + _state.FlowRegistration = default; + return DisposeRegistrationsAsync(callerRegistration, flowRegistration); + + static async ValueTask DisposeRegistrationsAsync( + CancellationTokenRegistration callerRegistration, + CancellationTokenRegistration flowRegistration) { - CompleteEnumerationWithException(close); - MarkBodyInitiatedDrain(); + await callerRegistration.DisposeAsync().ConfigureAwait(false); + await flowRegistration.DisposeAsync().ConfigureAwait(false); } } - // Publish progress unconditionally. A terminal delivery can lose its task-source CAS to an already- - // completed generation; a synchronous disposer must still observe this sticky level rather than park. - void SignalPumpProgress() - => _callerInteractionCore.SignalProgress(); + void DisposeRegistrations() + { + var cancellation = Volatile.Read(ref _state.ColdState); + var callerRegistration = cancellation?.CallerRegistration ?? default; + if (cancellation is not null) + cancellation.CallerRegistration = default; + var flowRegistration = _state.FlowRegistration; + _state.FlowRegistration = default; + callerRegistration.Dispose(); + flowRegistration.Dispose(); + } - void CompleteEnumerationWithException(Exception ex) + // Cancellation only latches intent and requests a backend cancel. The frame owning the decoder + // delivers it after the wire is back at RFQ. An idle flow drains autonomously first. + void RequestCancel(CancellationToken token, CommandExecutionCancellationScope scope, + BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, + BackendCancellationTiming subsequentTiming = BackendCancellationTiming.AfterGrace) { - // Close state must survive task-source rearming, including flows whose body never started. - if (ex is PgClientClosedException or PgCollateralException) - _callerInteractionCore.SetCloseLatch(ex); - if (IsEnumerationCompleted) + if (Volatile.Read(ref _state.Phase) == PhaseCompleted) return; - // Teardown may race the consumer. The task source is the completion authority; - // _enumeratorCompleted follows only when this call wins the current generation. - if (TrySetEnumeratorException(ex, runContinuationsAsynchronously: true)) - PublishEnumerationCompleted(); - // A faulted body will not publish another continuation. - SignalPumpProgress(); - // Wire recovery remains with the body or the framework recovery flow; this method only completes - // the consumer-facing generation. + var cancellation = GetOrCreateColdState(); + cancellation.DeliverToken = token; + RaiseCancellationScope(cancellation, scope); + RaiseCancellationTiming(ref cancellation.Timing, timing); + RaiseCancellationTiming(ref cancellation.SubsequentTiming, subsequentTiming); + Interlocked.Exchange(ref cancellation.CancelRequested, true); + if (HasDecoder) + RequestBackendCancellation(); + TryTakeOverDrain(); } - void PublishBodyTerminated() + static void RaiseCancellationScope(CommandExecutionColdState cancellation, CommandExecutionCancellationScope scope) { - Volatile.Write(ref _bodyState, BodyTerminated); - SignalPumpProgress(); + var requested = (int)scope; + var current = Volatile.Read(ref cancellation.Scope); + while (current < requested) + { + var observed = Interlocked.CompareExchange( + ref cancellation.Scope, requested, current); + if (observed == current) + return; + current = observed; + } } - bool TerminateBodyBeforeStart() - => Interlocked.CompareExchange(ref _bodyState, BodyTerminated, BodyNotStarted) == BodyNotStarted; - - bool IsBodyRunning => Volatile.Read(ref _bodyState) == BodyRunning; - bool IsBodyTerminated => Volatile.Read(ref _bodyState) == BodyTerminated; - - // Source handoff finishes before body/consumer rendezvous begins, so both reuse the same wait event. - private protected override FlowHandoffEvent? HandoffEvent => _callerInteractionCore.GetWaitEvent(); - - // Return the rendezvous directly. An async wrapper could signal the disposer before registering the - // body's continuation, causing a late ThreadPool dispatch instead of caller-thread drain execution. - FlowCallerInteractionCore.CallerHandoffAwaitable YieldToCaller() + static void RaiseCancellationTiming( + ref int location, BackendCancellationTiming timing) { - FieldRef> fieldRef; - unsafe + var requested = (int)timing; + var current = Volatile.Read(ref location); + while (current < requested) { - fieldRef = FieldRef>.Create(&GetCallerInteractionCore, this); + var observed = Interlocked.CompareExchange(ref location, requested, current); + if (observed == current) + return; + current = observed; } - return _callerInteractionCore.YieldToCaller(fieldRef); } - static ref FlowCallerInteractionCore GetCallerInteractionCore(CommandFlow instance) - => ref instance._callerInteractionCore; + internal Task CancelAsync() + { + var cancellation = GetOrCreateColdState(); + var delivery = Volatile.Read(ref cancellation.Delivery) + ?? Interlocked.CompareExchange(ref cancellation.Delivery, + new(TaskCreationOptions.RunContinuationsAsynchronously), null) + ?? cancellation.Delivery; + if (Volatile.Read(ref _state.Phase) is PhaseCompleted) + { + delivery.TrySetResult(); + return delivery.Task; + } + RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + if (Volatile.Read(ref _state.Phase) is PhaseCompleted) + delivery.TrySetResult(); + return delivery.Task; + } - protected override void OnAbort(Exception exception) => FaultCaller(exception); + void RequestBackendCancellation() + { + var cancellation = GetOrCreateColdState(); + var episodeKey = Volatile.Read(ref cancellation.EpisodeKey); + if (episodeKey is null) + { + var created = new object(); + episodeKey = Interlocked.CompareExchange( + ref cancellation.EpisodeKey, created, null) ?? created; + } + _state.Context.RequestBackendCancellation( + _ops.Flow, _ops.Flow.CancellationWindow, + (BackendCancellationTiming)Volatile.Read(ref cancellation.Timing), + Volatile.Read(ref cancellation.Delivery), + episodeKey, + Math.Max(Volatile.Read(ref cancellation.Scope), (int)CommandExecutionCancellationScope.CurrentWindow), + (BackendCancellationTiming)Volatile.Read(ref cancellation.SubsequentTiming)); + } - // Graceful stopping is the early wire-close wake and is idempotent across heartbeat ticks. - protected override void OnStopping(Exception exception) + // Graceful stop. An unactivated flow releases its consumer, the closing wire owns its response. + // An idle activated flow drains itself to RFQ so the pipeline can complete. + internal void OnStopping(Exception exception) { - if (!IsBodyRunning || !IsAsync) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); + if (CompleteReady(exception, runContinuationsAsynchronously: true)) { - FaultCaller(exception); + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); return; } - - // Resume normally so the body observes the close latch and drains; abort faults the gate. - _callerInteractionCore.SetCloseLatch(exception); - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); + TryTakeOverDrain(); } - // Wake a running body so it owns fault delivery; directly fault a flow whose body never started. - void FaultCaller(Exception exception) + // Forceful abort. No frame can read a dead wire, so an idle owner faults the pipeline task + // directly. A frame in flight fails on its own read. + internal void OnAbort(Exception exception) { - if (TerminateBodyBeforeStart()) + Interlocked.CompareExchange(ref GetOrCreateColdState().CloseException, exception, null); + if (CompleteReady(exception, runContinuationsAsynchronously: true)) { - CompleteEnumerationWithException(exception); - // A synchronous flow may already have entered ExecuteAfterHandoff while its inner read - // body is still NotStarted. Terminating that body does not complete the outer execution: - // fault and wake its initial handoff so the framework task can settle and release tenure. - _callerInteractionCore.FaultBodyWait(exception); - _callerInteractionCore.WakeBody(); + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); return; } + while (true) + { + var phase = Volatile.Read(ref _state.Phase); + if (phase is not (PhaseInitial or PhaseResultReady)) + return; + if (Interlocked.CompareExchange(ref _state.Phase, PhaseCompleted, phase) != phase) + continue; + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); + _state.PipelineTaskSource.SetException(exception, runContinuationsAsynchronously: true); + return; + } + } - // A concurrent body start may have beaten the pre-start terminal claim. - if (IsBodyRunning) - _callerInteractionCore.FaultBodyWait(exception); - else - CompleteEnumerationWithException(exception); + internal void Fail(Exception exception) + { + // A result callback failed on the frame that owns the decoder. Its throw propagates there. + Interlocked.CompareExchange(ref GetOrCreateColdState().TerminalException, exception, null); } - internal override void Fail(Exception exception) => FaultCaller(exception); + internal void OnReleasing(Exception? exception) + { + Volatile.Read(ref _state.ColdState)?.Delivery?.TrySetResult(); + DisposeRegistrations(); + _state.Commands.Return(); + } - protected override void OnReleasing(Exception? exception) + internal void OnDiscarded() { - if (Volatile.Read(ref _cancellationState) is { } cancellation) - Volatile.Read(ref cancellation.Delivery)?.TrySetResult(); - _commands.Return(); + _ops.OnDiscarded(); + _state.Commands.Return(); } - protected override void OnDiscarded() + internal void OnReset() { - // Discarded flows never enter the base release path. - GetObserver(out var observerState)?.OnCompleting(this, null, observerState); - _commands.Return(); + _state.Phase = PhaseInitial; + _state.CommandIndex = -1; + _state.Context = default; + _state.ContextPublished = false; + _state.Current = null; + _state.CurrentPublished = false; + _state.ReadFlowRfq = false; + _state.ConsumerDetached = false; + _state.ConsumerObservedCompletion = false; + _state.ReadySource.Reset(); + _state.PipelineTaskSource.Reset(); + _state.ReadyCompletion = 0; + _state.DrainStarted = 0; + _state.FlowToken = default; + _state.FlowRegistration = default; + _state.ColdState = null; + _state.SyncHandoffClaimed = false; + _state.HandoffEvent?.ResetInteraction(); + _state.WaitForDrainOnDispose = true; } - protected override void OnReset() +} + +public sealed partial class CommandFlow +{ + bool IValueTaskSource.GetResult(short token) => _state.ReadySource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _state.ReadySource.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _state.ReadySource.OnCompleted(continuation, state, token, flags); + + void IValueTaskSource.GetResult(short token) => _state.PipelineTaskSource.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _state.PipelineTaskSource.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _state.PipelineTaskSource.OnCompleted(continuation, state, token, flags); + + public readonly struct Enumerator : IEnumerator, IAsyncEnumerator { - Debug.Assert(IsPending || IsCompleted); - _commandIndex = -1; - _executePipelinedCore.Reset(); - ResetEnumeratorMoveNextSource(); - _enumeratorCurrent = default; - _enumeratorCompleted = false; - _isResultReady = false; - _callerInteractionCore.Reset(); - if (_cancellationState is { } cancellation) - { - cancellation.Reset(); - _cancellationState = null; - } - _drainErrors = null; - _consumeNonQuery = false; - _nonQueryRecordsAffected = 0; - _consumerDisposed = false; - _draining = false; - _drainModeEntered = false; - WaitForDrainOnDispose = true; - // Dispatch state is per-tenure. - _contextPublished = false; - _context = default; - _bodyState = BodyNotStarted; - _consumerAdvanced = false; - } - - FlowCallerInteractionCoreResult IValueTaskSource.GetResult(short token) - => _callerInteractionCore.ConsumeGateResult(token); - - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) - => _callerInteractionCore.GateStatus(token); - - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - { - _callerInteractionCore.OnGateCompleted(continuation, state, token, flags); - // Drain is a sticky level. Recheck it after registering so an earlier gate edge cannot be lost - // across gate reset. This is the body's suspending stack, so the wake must dispatch asynchronously. - if (IsDraining) + readonly CommandFlow? _flow; + readonly CancellationToken _cancellationToken; + + public Enumerator(CommandFlow flow) + : this(flow, default) + { } + + internal Enumerator(CommandFlow flow, CancellationToken cancellationToken) { - // A synchronous takeover would already have resumed the body inline. Reaching this callback - // means autonomous execution still owns the body. - IsAsync = true; - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); + _flow = flow; + _cancellationToken = cancellationToken; } - } - // Backing for the pipelined-dispatch ValueTask. Returned to the framework when activation - // hasn't fired yet. Nested callback completes it when ExecutePipelined finishes. - void IValueTaskSource.GetResult(short token) => _executePipelinedCore.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _executePipelinedCore.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - => _executePipelinedCore.OnCompleted(continuation, state, token, flags); + public Enumerator GetAsyncEnumerator() => this; + + public Enumerator GetEnumerator() => this; + + public bool MoveNext() => _flow?.Core.MoveNext() ?? false; + + public ValueTask MoveNextAsync() => MoveNextAsync(_cancellationToken); + public ValueTask MoveNextAsync(CancellationToken cancellationToken) + => _flow is null ? new(false) : _flow.Core.MoveNextAsync(cancellationToken); + + public CommandResult Current => _flow?._state.Current ?? default!; + + object? IEnumerator.Current => Current; + + void IEnumerator.Reset() => throw new NotSupportedException(); + + public ValueTask DisposeAsync() => _flow is null ? default : _flow.Core.DisposeAsync(); + + public void Dispose() => _flow?.Core.Dispose(); + } } diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs b/Slon/Pg/Protocol/Flows/LegacyCommandFlow.Enumerator.cs similarity index 98% rename from Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs rename to Slon/Pg/Protocol/Flows/LegacyCommandFlow.Enumerator.cs index 78eb24b..fc07e38 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.Enumerator.cs +++ b/Slon/Pg/Protocol/Flows/LegacyCommandFlow.Enumerator.cs @@ -3,7 +3,7 @@ namespace Slon.Pg.Protocol.Flows; -partial class CommandFlow +partial class LegacyCommandFlow { Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _enumeratorMoveNextTaskSource; int _enumeratorMoveNextCompletionClaim; @@ -161,7 +161,7 @@ void EnsureEnumerationCompleted() } } - public readonly struct Enumerator(CommandFlow flow) : IEnumerator, IAsyncEnumerator + public readonly struct Enumerator(LegacyCommandFlow flow) : IEnumerator, IAsyncEnumerator { // Here so we can pass the cancellation token and enumerate without boxing the struct (which WithCancellation must do). /// @@ -375,7 +375,7 @@ public void Dispose() if (flow.WaitForDrainOnDispose) flow.AwaitDrainOnDisposeSynchronously(); - static void DriveBodyToTermination(CommandFlow flow) + static void DriveBodyToTermination(LegacyCommandFlow flow) { while (!flow.IsBodyTerminated) { @@ -392,7 +392,7 @@ static void DriveBodyToTermination(CommandFlow flow) } } - static void FinishCompletedDisposal(CommandFlow flow) + static void FinishCompletedDisposal(LegacyCommandFlow flow) { if (flow.TransferLiveBodyToDrain() && flow.WaitForDrainOnDispose) flow.AwaitDrainOnDisposeSynchronously(); @@ -421,7 +421,7 @@ public ValueTask DisposeAsync() return flow.AwaitDrainOnDispose(); return new(); - static async ValueTask FinishFinalResultAsync(Enumerator enumerator, CommandFlow flow) + static async ValueTask FinishFinalResultAsync(Enumerator enumerator, LegacyCommandFlow flow) { if (await enumerator.MoveNextAsync().ConfigureAwait(false)) ThrowHelper.ThrowInvalidOperation( @@ -429,7 +429,7 @@ static async ValueTask FinishFinalResultAsync(Enumerator enumerator, CommandFlow await FinishCompletedDisposalAsync(flow).ConfigureAwait(false); } - static ValueTask FinishCompletedDisposalAsync(CommandFlow flow) + static ValueTask FinishCompletedDisposalAsync(LegacyCommandFlow flow) { // A completed awaited drain may still have errors to surface. flow.TransferLiveBodyToDrain(); diff --git a/Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs b/Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs new file mode 100644 index 0000000..33948de --- /dev/null +++ b/Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs @@ -0,0 +1,1331 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks.Sources; +using Slon.Runtime.CompilerServices; +// A unique result type distinguishes the caller gate from this flow's other IValueTaskSource instantiations. +using FlowCallerInteractionCoreResult = System.ValueTuple; + +namespace Slon.Pg.Protocol.Flows; + +[Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] +internal abstract class LegacyCommandFlowObserver : PgClientFlowObserver +{ + protected internal virtual void OnStarted(LegacyCommandFlow flow, object? state) { } + protected internal virtual void OnCommandResult(LegacyCommandFlow flow, CommandResult result, object? state) { } + protected internal virtual void OnDrainStarted(LegacyCommandFlow flow, object? state) { } +} + +[Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] +internal readonly struct LegacyCommandFlowOptions +{ + public LegacyCommandFlowObserver? Observer { get; init; } + public object? ObserverState { get; init; } + public CommandList Commands { get; init; } + // Optional per-flow override for time spent waiting in the protocol backlog. + public TimeSpan? PendingTimeout { get; init; } +} + +[Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] +internal partial class LegacyCommandFlow : PgClientFlow, IValueTaskSource, IValueTaskSource, IValueTaskSource +{ + sealed class PreparationReadState + { + internal ParameterTypeList ParameterTypes; + } + + static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); + + internal override bool DefersSyncHandoff => true; + + internal enum CancellationScope : byte + { + None, + CurrentWindow = 1, + RemainingFlow = 2 + } + + // Flow state + CommandList _commands; + TimeSpan? _pendingTimeout; + FlowCallerInteractionCore _callerInteractionCore; + // Cancellation is cold; keep its tokens, registrations and attribution state off ordinary flows. + CancellationState? _cancellationState; + // Errors encountered while draining are surfaced by a waiting DisposeAsync. Live consumers observe + // their errors directly, and the list remains unallocated on the successful path. + List? _drainErrors; + // Set only by ConsumeNonQueryAsync, after enqueue but before any consumer-side gate release. + // The body cannot reach first publication until such a release, and every release publishes + // this write, so plain accesses suffice and the body observes the mode at first wake. + bool _consumeNonQuery; + bool IsConsumingNonQuery => _consumeNonQuery; + bool IsConsumingAutonomously => IsDraining || IsConsumingNonQuery; + long _nonQueryRecordsAffected; + + // Once draining, the body bypasses result handoffs and reads autonomously to RFQ. This is state, not + // an I/O cancellation token: canceling the I/O would prevent restoration of a clean wire boundary. + bool _draining; + internal bool IsDraining => Volatile.Read(ref _draining); + // Body-thread-only guard: later commands must not change the drive mode chosen on drain entry. + bool _drainModeEntered; + + + // Distinguishes consumer disposal from a body-initiated drain; disposal suppresses terminal OCE delivery. + bool _consumerDisposed; + // When true, DisposeAsync awaits the body's drain to RFQ. Otherwise it returns while the body drains; + // pipeline retirement still prevents the next flow from observing a dirty wire. + internal bool WaitForDrainOnDispose { get; set; } = true; + + // Consumer disposal without waiting for the autonomous drain. + void MarkConsumerGone() + { + Volatile.Write(ref _consumerDisposed, true); + RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, + BackendCancellationTiming.AtReadFrontier); + } + + // Consumer disposal while waiting for the autonomous drain. + void MarkConsumerWaitForDrain() + { + Volatile.Write(ref _consumerDisposed, true); + RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, + BackendCancellationTiming.AtReadFrontier); + } + + // A synchronous consumer cannot abandon its drive obligation while the body is parked. Give the + // cancellation path a delivery source so WakeBody transfers that obligation to the dedicated driver. + void MarkSyncConsumerGone() + { + Volatile.Write(ref _consumerDisposed, true); + _ = GetOrCreateCancelDelivery(); + RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, + BackendCancellationTiming.AtReadFrontier); + _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); + _callerInteractionCore.WakeBody(useDedicatedDriver: true); + } + + // Enumeration already ended; only transfer the live body tail to autonomous ownership. + // There is no unfinished consumer work to justify a backend cancellation intent. + void MarkTerminalConsumerGone() + { + Volatile.Write(ref _consumerDisposed, true); + Volatile.Write(ref _draining, true); + } + + // Result publication orders these body-owned fields before the consumer can observe the + // CommandResult. IsComplete is consumer-owned after that handoff. A behavior-limited reader may + // consider its visible result final while later commands still exist, so use the physical + // command index rather than an ADO-visible result count. + bool IsFullyConsumedFinalResult + => _isResultReady && _commandIndex >= CommandCount - 1 + && _enumeratorCurrent is { IsComplete: true }; + + // A body-initiated drain keeps the consumer attached for terminal cancellation or close delivery. + void MarkBodyInitiatedDrain() => Volatile.Write(ref _draining, true); + + // Result-production state + RowDescription? _requestedRowDescription; + PgError? _pgError; + int _commandIndex = -1; + PgDecoder? _decoder; + bool _readFlowRfq; + + // Pipelined dispatch state. Lives here (not on PgClientFlow base) because the shared-promise + // optimization that needs these fields is LegacyCommandFlow-specific (see DispatchPipelinedRead). + Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _executePipelinedCore; + Context _context; + bool _contextPublished; + // Consumer terminality, body terminality, and framework release are distinct phases. Start and + // pre-start termination race during shutdown, so one atomic state owns that decision. + const int BodyNotStarted = 0; + const int BodyRunning = 1; + const int BodyTerminated = 2; + int _bodyState; + DetachedPublication _detachedPublication; + + enum DetachedPublication : byte + { + None, + Result, + Completion + } + + sealed class CancellationState + { + internal CancellationToken CallerToken; + internal CancellationToken FlowToken; + internal CancellationTokenRegistration CallerRegistration; + internal CancellationTokenRegistration FlowRegistration; + internal bool Requested; + internal int Scope; + internal int Timing; + internal int SubsequentTiming; + internal CancellationToken DeliverToken; + internal TaskCompletionSource? Delivery; + internal object? EpisodeKey; + internal bool DeliverOce; + + internal void Reset() + { + CallerToken = default; + FlowToken = default; + CallerRegistration.Dispose(); + CallerRegistration = default; + FlowRegistration.Dispose(); + FlowRegistration = default; + Requested = false; + Scope = (int)CancellationScope.None; + Timing = (int)BackendCancellationTiming.AfterGrace; + SubsequentTiming = (int)BackendCancellationTiming.AfterGrace; + DeliverToken = default; + Delivery = null; + EpisodeKey = null; + DeliverOce = false; + } + } + LegacyCommandFlow() : base(supportsDeferredFlush: true) + { + _callerInteractionCore.Initialize(); + } + + // Interactive commands carry caller patience, so arm the activation timeout. + protected override bool EnableActivationTimeout => true; + protected override TimeSpan? PendingTimeout => _pendingTimeout; + internal override TimeSpan? BackendCancellationGracePeriod + => Volatile.Read(ref _consumerDisposed) ? ConsumerDrainCancellationGracePeriod : null; + + internal LegacyCommandFlow(bool async, params ReadOnlySpan commands) : this() + => Initialize(async, commands); + internal LegacyCommandFlow(bool async, in LegacyCommandFlowOptions options) : this() + => Initialize(async, options); + + private protected LegacyCommandFlow(bool async, TimeSpan? pendingTimeout) : this() + { + IsAsync = async; + _pendingTimeout = pendingTimeout; + } + + internal LegacyCommandFlow Initialize(bool async, params ReadOnlySpan commands) + => Initialize(async, options: new() { Commands = new(commands) }); + + internal LegacyCommandFlow Initialize(bool async, in LegacyCommandFlowOptions options) + { + IsAsync = async; + if (options.Observer is { } observer) + SetObserver(observer, options.ObserverState); + _commands = options.Commands; + _pendingTimeout = options.PendingTimeout; + options.Observer?.OnStarted(this, options.ObserverState); + return this; + } + + // Declares internal non-query consumption and runs it to completion. The single entry point + // makes the ownership rule structural: no enumerator is exposed on this path, so mixing + // enumeration with internal consumption is unrepresentable. The declaration precedes any + // consumer-side gate release, so the body observes it at first wake and never publishes. + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + internal async ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) + { + _consumeNonQuery = true; + _nonQueryRecordsAffected = -1; + var enumerator = GetAsyncEnumerator(cancellationToken); + try + { + // Release a body parked pre-publication and wake the flow. + _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); + _callerInteractionCore.WakeBody(); + _ = await EnumeratorMoveNextTask.ConfigureAwait(false); + // Internal consumption owns error delivery. Errors collected during the internal + // drain have no other outlet: DisposeAsync deliberately skips a completed flow. + if (_drainErrors is { Count: > 0 } errors) + throw errors.Count == 1 ? errors[0] : new AggregateException(errors); + return _nonQueryRecordsAffected; + } + finally + { + await enumerator.DisposeAsync().ConfigureAwait(false); + } + } + + // The token in force for the current read: the flow token once fired (whole-flow cancel), else the + // per-read token if cancelable, else the flow token. + CancellationToken EffectiveCancellationToken + => GetEffectiveCancellationToken(Volatile.Read(ref _cancellationState)); + + static CancellationToken GetEffectiveCancellationToken(CancellationState? cancellation) + => cancellation is null ? default + : cancellation.FlowToken.IsCancellationRequested ? cancellation.FlowToken + : cancellation.CallerToken.CanBeCanceled ? cancellation.CallerToken + : cancellation.FlowToken; + + public int CommandCount => _commands.Count; + internal virtual int VisibleCommandCount => _commands.VisibleCount; + public bool IsResultReady => _isResultReady; + + public Enumerator GetEnumerator() + { + return new Enumerator(this); + } + + // Bind at submission because eager writing precedes the first MoveNextAsync. + internal override void BindCallerToken(CancellationToken cancellationToken) + => GetOrCreateCancellationState().FlowToken = cancellationToken; + internal override CancellationToken MigrationCancellationToken + => Volatile.Read(ref _cancellationState)?.FlowToken ?? default; + + public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + // A missing per-call token must not replace the flow token captured at submission. + if (cancellationToken.CanBeCanceled) + GetOrCreateCancellationState().FlowToken = cancellationToken; + return new(this); + } + + protected override ValueTask ExecuteAuto(Context context) + { + if (!IsAsync && _callerInteractionCore.IsWaiting) + return ExecuteAfterHandoff(context); + + return new(ExecuteAutoCore(context)); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask ExecuteAfterHandoff(Context context) + { + try + { + await YieldToCaller(); + } + catch (Exception ex) + { + TerminateBodyBeforeStart(); + CompleteEnumerationWithException(ex); + throw; + } + + return ExecuteAutoCore(context); + } + + FlowTasks ExecuteAutoCore(Context context) + { + _context = context; + Volatile.Write(ref _contextPublished, true); + if (Volatile.Read(ref _cancellationState) is { } cancellation) + { + if (cancellation.FlowToken.IsCancellationRequested) + RequestCancel(cancellation.FlowToken, CancellationScope.RemainingFlow); + else if (Volatile.Read(ref cancellation.Requested)) + RequestBackendCancellation((BackendCancellationTiming)Volatile.Read(ref cancellation.Timing), + Volatile.Read(ref cancellation.Delivery)); + } + ValueTask writeTask; + try + { + // Writes are independent of consumer admission. Inter-result gates provide backpressure. + // Async writes use transport completion; sync writes use the resumable non-blocking path so + // the caller thread retains execution ownership across readiness waits. + var appendSync = !_commands[CommandCount - 1].WithSync; + _readFlowRfq = appendSync; + // Caller cancellation never cancels wire I/O. A partially cancelled write requires + // protocol recovery and can strand already-pipelined successors; the body instead + // observes the latched intent and drains every written command to RFQ. + writeTask = IsAsync + ? _commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) + : WriteCommandsResumable(context, appendSync); + + // Observe synchronous faults here; pending writes remain the framework-owned trailing task. + if (writeTask.IsCompleted) + writeTask.GetAwaiter().GetResult(); + } + catch (Exception ex) + { + TerminateBodyBeforeStart(); + CompleteEnumerationWithException(ex); + throw; + } + + // Read and write run concurrently; the framework observes the trailing write before releasing + // the flow, preserving single-writer tenure without blocking reads behind socket backpressure. + return new FlowTasks( + trailingExecutionTask: writeTask, + pipelineTask: DispatchPipelinedRead( + context, context.GetProtocolStatic().Promise)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + ValueTask WriteCommandsResumable(Context context, bool appendSync) + { + var encoder = context.GetEncoder(); + ValueTask writeTask; + using (encoder.BeginResumableWriteScope()) + writeTask = _commands.WriteCommandsResumable(encoder, appendSync); + return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); + } + + // Defer state-machine creation until activation because all flows share one protocol-static promise. + ValueTask DispatchPipelinedRead(Context context, ValueTaskSourcePromise promise) + { + // The shared promise may be tenured only after successful decoder activation. + var waiter = context.GetDecoderAsync().ConfigureAwait(false); + if (waiter.IsCompleted) + { + // Only successful activation owns the shared promise. A settled fault belongs to this flow's + // private completion source because it never claimed the wire. + if (!waiter.IsCompletedSuccessfully) + { + // Preserve the activation's close, timeout, or cancellation identity. + try { waiter.GetAwaiter().GetResult(); } + catch (Exception ex) { _executePipelinedCore.SetException(ex); } + return new ValueTask(this, _executePipelinedCore.Version); + } + // Handing the shared-promise-backed task to the framework is safe: the contract guarantees + // the waiter is consumed (releasing the promise tenure via GetResult's Reset) before the + // item's position is republished, so a successor's dispatch always finds the tenure released. + PromiseAsyncValueTaskMethodBuilder.Promise = promise; + try + { + return ExecutePipelined(context); + } + finally + { + PromiseAsyncValueTaskMethodBuilder.Promise = null; + } + } + + // Static continuation: a bridge into framework state, so no captured scheduling context is needed. + waiter.OnCompleted(static state => + { + var flow = (LegacyCommandFlow)state!; + var ctx = flow._context; + // A faulted activation never claimed the shared promise; complete only this flow's source. + var activation = ctx.GetDecoderAsync().GetAwaiter(); + if (!activation.IsCompletedSuccessfully) + { + // Preserve the activation's close, timeout, or cancellation identity. + try { activation.GetResult(); } + catch (Exception ex) { flow._executePipelinedCore.SetException(ex); } + return; + } + var promise = ctx.GetProtocolStatic().Promise; + PromiseAsyncValueTaskMethodBuilder.Promise = promise; + ValueTask task = flow.ExecutePipelined(ctx); + try + { + if (!task.IsCompleted) + { + ((IValueTaskSource)promise).OnCompleted(static state => + { + var flow = (LegacyCommandFlow)state!; + try + { + var promise = flow._context + .GetProtocolStatic().Promise; + ((IValueTaskSource)promise).GetResult(promise.Token); + flow._executePipelinedCore.SetResult(true); + } + catch (Exception ex) + { + flow._executePipelinedCore.SetException(ex); + } + // This internal bridge runs no user code and requires no ExecutionContext flow. + }, flow, promise.Token, ValueTaskSourceOnCompletedFlags.None); + } + else + { + try + { + task.GetAwaiter().GetResult(); + flow._executePipelinedCore.SetResult(true); + } + catch (Exception ex) + { + flow._executePipelinedCore.SetException(ex); + } + } + } + finally + { + PromiseAsyncValueTaskMethodBuilder.Promise = null; + } + }, this); + + return new ValueTask(this, _executePipelinedCore.Version); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder))] + async ValueTask ExecutePipelined(Context context) + { + // Pre-start teardown may have already claimed terminality. In that case the consumer has its + // fault and this late dispatch has no body tenure to establish. + if (Interlocked.CompareExchange(ref _bodyState, BodyRunning, BodyNotStarted) != BodyNotStarted) + return; + try + { + // If we have a continuation stored we must already be on the caller thread, + // otherwise we must make sure to unblock the executor (see comment in the write phase). + // This first handoff is body execution too: close may fault it, so it belongs inside the + // same terminal envelope that publishes the consumer fault and body termination. + if (!IsAsync && !_callerInteractionCore.HasHandoff) + await YieldToCaller(); + + // User cancellation must not cancel activation or wire I/O. The flow observes it after + // activation, drains itself to RFQ, then delivers OCE without invoking pipeline recovery. + _decoder = await context.GetDecoderAuto().ConfigureAwait(false); + var publishedResult = false; + while (++_commandIndex < CommandCount) + { + _isResultReady = false; + bool hasPreparedDescription; + bool suppressEnumeration; + bool describeForPreparation; + { + ref readonly var command = ref _commands.ItemRef(_commandIndex); + _decoder.UseReadTimeout(command.Timeout); + suppressEnumeration = command.SuppressEnumeration; + describeForPreparation = command.DescribeForPreparation; + hasPreparedDescription = command.Descriptor is { IsPrepared: true, PreparedRowDescription: not null } + && !command.DescribeOnly; + } + + // Registrations only latch and wake; terminal delivery remains body-owned. Dispose them + // before promise tenure ends so callbacks cannot reach the next flow. Do not rearm after + // consumer disposal, where a persistent cancellation could escape its intended wait. + if (Volatile.Read(ref _cancellationState) is { } cancellationAtReadStart && !IsDraining + && (cancellationAtReadStart.CallerToken.CanBeCanceled + || cancellationAtReadStart.FlowToken.CanBeCanceled)) + RegisterCancellationCallbacks(cancellationAtReadStart); + // After close, a fresh command must not consume bytes left by its predecessor. A draining + // flow may continue reading its own response to restore RFQ. + if (!IsDraining && context.IsProtocolClosed) + throw context.FlowTerminationException; + + PreparationReadState? preparationRead = null; + if (describeForPreparation) + { + preparationRead = new(); + await ReadPreparationDescription(context, preparationRead).ConfigureAwait(false); + } + else if (IsAsync && hasPreparedDescription) + { + // Prepared commands with a known description have the compact BindComplete -> + // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes + // this outer body rather than a nested parser coroutine; the second message normally + // comes from the same batch and is consumed synchronously. + if (!_decoder.TryMoveNext()) + { + if (!await _decoder.MoveNextAsync().ConfigureAwait(false)) + _decoder.ThrowUnexpectedEof(); + } + var message = _decoder.Current; + + if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) + { + _pgError = bindError; + _requestedRowDescription = null; + } + else + { + if (!_decoder.TryMoveNext()) + { + if (!await _decoder.MoveNextAsync().ConfigureAwait(false)) + _decoder.ThrowUnexpectedEof(); + } + message = _decoder.Current; + message.DebugEnsureExpected(PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); + _pgError = null; + _requestedRowDescription = null; + } + } + else if (IsAsync) + { + var rowDescription = context.GetProtocolStatic().RowDescription; + var read = _commands.ItemRef(_commandIndex).ReadUntilExecuteAsync(_decoder, rowDescription); + (_pgError, _requestedRowDescription) = await read.ConfigureAwait(false); + } + else + { + var rowDescription = context.GetProtocolStatic().RowDescription; + (_pgError, _requestedRowDescription) = _commands.ItemRef(_commandIndex) + .ReadUntilExecute(_decoder, rowDescription); + } + + // A draining consumer cannot observe CommandResult, so retain read errors for disposal. + var capturedThisCommand = false; + var readErrorIsOwnCancellation = _pgError is { } readError + && IsOwnCancellation(readError); + if (IsConsumingAutonomously && _pgError is { } readErrorToCapture + && !readErrorIsOwnCancellation) + { + (_drainErrors ??= new()).Add(PgErrorException.Create(readErrorToCapture)); + capturedThisCommand = true; + } + + // Await in-flight callbacks before releasing shared promise tenure. + var cancellation = Volatile.Read(ref _cancellationState); + if (cancellation is not null) + await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); + // Cancellation switches to autonomous drain; terminal delivery follows RFQ and tenure release. + var effectiveCancellationToken = GetEffectiveCancellationToken(cancellation); + if ((cancellation is { } && Volatile.Read(ref cancellation.Requested) + || effectiveCancellationToken.IsCancellationRequested) && !IsEnumerationCompleted) + { + cancellation ??= GetOrCreateCancellationState(); + if (effectiveCancellationToken.IsCancellationRequested) + cancellation.DeliverToken = effectiveCancellationToken; + cancellation.DeliverOce = true; + if (!IsDraining) + MarkBodyInitiatedDrain(); + } + + var result = InitializeResult( + context, preparationRead); + ((LegacyCommandFlowObserver?)GetObserver(out var observerState)) + ?.OnCommandResult(this, result, observerState); + + // Disposal drains without another result handoff. Graceful close instead faults the + // attached consumer, then uses the same autonomous drain. Command errors remain results. + if (context.StoppingToken.IsCancellationRequested && !IsDraining + && !IsEnumerationCompleted) + { + // Latch the close (a consumer that Resets past this point self-delivers it), wake a + // parked consumer, then drain. + var close = context.FlowTerminationException; + _callerInteractionCore.SetCloseLatch(close); + CompleteEnumerationWithClose(close); + MarkBodyInitiatedDrain(); + } + if (!IsDraining && !IsConsumingNonQuery && !suppressEnumeration) + { + // Eager async execution must wait for the consumer to arm generation zero before + // publishing its first result. Synchronous execution already runs on that caller. + if (!publishedResult && IsAsync) + { + await _callerInteractionCore.WaitForCaller(this).ConfigureAwait(false); + // The first consumer can arrive after the response prelude was already read + // and its registrations were disposed. Its token was armed by MoveNextAsync; + // the result has now won that race, so retire the late registration before + // publishing the result. + if (Volatile.Read(ref _cancellationState) is { } lateCancellation) + await DisposeCancellationRegistrations(lateCancellation).ConfigureAwait(false); + EnterStoppingDrainIfNeeded(context); + } + + if (!IsDraining && !IsConsumingNonQuery) + { + _isResultReady = true; + publishedResult = true; + // Result continuations run asynchronously so the body can reach the next gate + // before user code asks for the next result. Buffered batches then advance inline + // from MoveNextAsync instead of suspending one Task state machine per result. + PublishEnumeratorResult(context, result); + + if (!IsDraining && !IsConsumingNonQuery) + { + if (IsAsync) + { + await _callerInteractionCore.WaitForCaller(this).ConfigureAwait(false); + EnterStoppingDrainIfNeeded(context); + } + else + await YieldToCaller(); + + /* The next MoveNext or MoveNextAsync call resumes here. */ + } + } + } + else if (!_drainModeEntered && IsAsyncAtDispatch && !IsAsync) + { + // An async I/O wake raced a synchronous disposer before the body reached its handoff. + if (WaitForDrainOnDispose) + { + // Hand the continuation to the disposer, which waits on the rendezvous rather than + // this task and can therefore drive the remaining drain without sync-over-async. + await YieldToCaller(); + } + else + { + // No disposer is waiting to drive; retain asynchronous background draining. + IsAsync = IsAsyncAtDispatch; + } + } + // Preserve the drive mode chosen on first drain entry. + _drainModeEntered = _drainModeEntered || IsDraining; + + // Disposing the message enumerator completes the command and, in drain mode, consumes its + // remaining rows. Re-read the current execution mode after every resumption. + (PgError Error, TransactionStatus TransactionStatus)? completeError; + // Consumption mode may change while the body is suspended; use the current value. + if (IsConsumingNonQuery || suppressEnumeration) + { + await CompleteInternalConsumptionAsync( + result, suppressEnumeration, capturedThisCommand).ConfigureAwait(false); + continue; + } + else if (IsAsync) + { + var resultEnumerator = context.GetProtocolStatic().ResultMessageEnumerator; + await resultEnumerator.DisposeAsync().ConfigureAwait(false); + completeError = resultEnumerator.CompleteError; + } + else + { + var resultEnumerator = context.GetProtocolStatic().ResultMessageEnumerator; + resultEnumerator.Dispose(); + completeError = resultEnumerator.CompleteError; + } + + if (result.Error is not null || completeError is not null) + await HandleCommandErrorsAsync( + result, suppressEnumeration, consumeInternally: false, + capturedThisCommand, completeError).ConfigureAwait(false); + } + + // The framework observes trailing write failure before releasing this flow. + if (_readFlowRfq) + { + if (_decoder.TryMoveNext()) + { + var message = _decoder.Current; + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } + else if (IsAsync) + { + await ReadRfqAsync(_decoder).ConfigureAwait(false); + } + else + { + ReadRfq(_decoder); + } + } + + PublishEnumeratorResult(context, null); + } + catch (PgClientClosedException) when (context.IsProtocolClosed) + { + // Scope to our own closure so a nested protocol's close doesn't get treated as ours. + // Latch the close so a consumer that Resets after this point self-delivers it. + _callerInteractionCore.SetCloseLatch(context.FlowTerminationException); + // A detached consumer treats close as drain completion; a live consumer observes the close. + if (IsDraining) + { + if (!IsEnumerationCompleted) + PublishEnumeratorResult(context, null); + return; + } + CompleteEnumerationWithException(context.FlowTerminationException); + throw; + } + catch (OperationCanceledException ex) when (IsCancellationToken(ex.CancellationToken)) + { + CompleteEnumerationWithException(ex); + throw; + } + catch (TimeoutException ex) + { + await HandleTimeoutAsync(context, ex).ConfigureAwait(false); + return; + } + catch (Exception ex) + { + CompleteEnumerationWithException(ex); + throw; + } + finally + { + // The body is the sole owner of protocol-static row metadata. Recovery consumes only + // decoder/wire state, so a faulted body can release oversized storage while recovery + // retains the failed flow's framework tenure. + ref readonly var readState = ref context.GetProtocolStatic(); + readState.Reset(); + PublishBodyTerminated(); + } + } + + void PublishEnumeratorResult(Context context, CommandResult? next) + { + var completed = next is null; + var publishAsync = IsAsync; + if (completed) + { + _enumeratorCurrent = null; + } + else + { + if (Volatile.Read(ref _cancellationState) is { } cancellation) + cancellation.CallerToken = default; + + if (!ReferenceEquals(_enumeratorCurrent, next)) + _enumeratorCurrent = next; + } + + // Close is durable across generations; complete the current one without publishing a result. + if (_callerInteractionCore.CloseException is not null) + { + CompleteEnumerationWithClose(_callerInteractionCore.CloseException); + return; + } + if (completed) + { + // Publish durable terminal state atomically with respect to consumer rearming. Async + // consumers complete from the protocol scheduler so they cannot reenter this lock or + // the pipeline frame that still owns the shared promise; sync consumers retain their + // caller-driven completion. + using (_rearmLock.EnterScope()) + { + PublishEnumerationCompleted(); + if (!publishAsync) + CompleteEnumeration(); + } + if (publishAsync) + SubmitPublication(context, DetachedPublication.Completion); + return; + } + if (publishAsync) + { + // Queue the publication itself so the body reaches its next caller gate before user code + // resumes. Routing through the protocol scheduler preserves that ordering without forcing + // every result continuation onto the ThreadPool. + SubmitPublication(context, DetachedPublication.Result); + } + else + TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); + } + + async ValueTask HandleTimeoutAsync(Context context, TimeoutException exception) + { + CompleteEnumerationWithException(exception); + RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, + BackendCancellationTiming.AtReadFrontier, allowCompletedEnumeration: true); + if (context.IsProtocolClosed) + ExceptionDispatchInfo.Throw(exception); + + // The timeout is terminal for the consumer, not for the body. Keep ownership of the + // command sequence and drain every remaining RFQ window; reaching each RFQ requests + // cancellation for each following window through the cancellation coordinator. Recovery is + // reserved for a failure of this semantic drain, where only wire obligations remain. + if (Volatile.Read(ref _cancellationState) is { } cancellation) + await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); + ((LegacyCommandFlowObserver?)GetObserver(out var observerState)) + ?.OnDrainStarted(this, observerState); + try + { + while (context.OutstandingRfqCount != 0) + _ = await _decoder!.GetNextAuto().ConfigureAwait(false); + } + catch (TimeoutException) + { + // The semantic drain owns the same cancellation episode. A timeout here would otherwise + // leave the episode unaware that its first read-timeout escalation made no protocol progress. + RequestCancel(default, CancellationScope.RemainingFlow, + BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier, + allowCompletedEnumeration: true); + throw; + } + } + + static async ValueTask ReadRfqAsync(PgDecoder decoder) + { + var message = await decoder.GetNextAsync().ConfigureAwait(false); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } + + static void ReadRfq(PgDecoder decoder) + { + var message = decoder.GetNext(); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) + PgErrorException.Throw(rfqError); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + CommandResult InitializeResult( + Context context, PreparationReadState? preparationRead) + { + ref readonly var readState = ref context.GetProtocolStatic(); + readState.ResultMessageEnumerator.Initialize(_commands.ItemRef(_commandIndex), _decoder!); + var result = _enumeratorCurrent ?? readState.CommandResult; + + ref readonly var command = ref _commands.ItemRef(_commandIndex); + var descriptor = command.Descriptor; + // We were preparing and we have no error from parse, make a prepared descriptor. + if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault + && (_pgError is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) + { + descriptor = CommandDescriptor.CreatePrepared( + descriptor.CommandName, + preparationRead?.ParameterTypes ?? descriptor.ParameterTypes, + _requestedRowDescription?.Preserve()); + } + result.Initialize(this, _commandIndex, descriptor, _requestedRowDescription, + !command.DescribeOnly, command.IsSimple(), _pgError); + return result; + } + + ValueTask ReadPreparationDescription(Context context, PreparationReadState state) + { + var rowDescription = context.GetProtocolStatic().RowDescription; + ref readonly var command = ref _commands.ItemRef(_commandIndex); + if (!IsAsync) + { + (_pgError, state.ParameterTypes, _requestedRowDescription) = + command.ReadPreparationDescription(_decoder!, rowDescription); + return default; + } + + var read = command.ReadPreparationDescriptionAsync(_decoder!, rowDescription); + if (!read.IsCompletedSuccessfully) + return AwaitRead(this, state, read); + (_pgError, state.ParameterTypes, _requestedRowDescription) = read.Result; + return default; + + static async ValueTask AwaitRead( + LegacyCommandFlow flow, PreparationReadState state, + ValueTask<(PgError?, ParameterTypeList, RowDescription?)> read) + { + (flow._pgError, state.ParameterTypes, flow._requestedRowDescription) = + await read.ConfigureAwait(false); + } + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] + async ValueTask CompleteInternalConsumptionAsync( + CommandResult result, bool suppressEnumeration, bool capturedThisCommand) + { + while (_decoder!.Current.Header.Type is PgTypes.BackendType.DataRow) + { + if (!_decoder.TryMoveNext()) + await _decoder.GetNextAsync().ConfigureAwait(false); + } + result.CompleteNonQuery(_decoder.Current); + var completeError = await _commands.ItemRef(_commandIndex) + .CompleteAsync(_decoder).ConfigureAwait(false); + if (_pgError is null && completeError is null) + { + var recordsAffected = result.GetCommandComplete().BatchRecordsAffected; + if (recordsAffected >= 0) + _nonQueryRecordsAffected = _nonQueryRecordsAffected < 0 + ? recordsAffected + : checked(_nonQueryRecordsAffected + recordsAffected); + } + + if (result.Error is not null || completeError is not null) + await HandleCommandErrorsAsync( + result, suppressEnumeration, consumeInternally: true, + capturedThisCommand, completeError).ConfigureAwait(false); + } + + async ValueTask HandleCommandErrorsAsync( + CommandResult result, bool suppressEnumeration, bool consumeInternally, + bool capturedThisCommand, + (PgError Error, TransactionStatus TransactionStatus)? completeError) + { + var resultErrorIsOwnCancellation = result.Error is { } resultError + && IsOwnCancellation(resultError); + if (suppressEnumeration && result.Error is { } suppressedError + && !resultErrorIsOwnCancellation) + { + (_drainErrors ??= new()).Add(PgErrorException.Create(suppressedError)); + capturedThisCommand = true; + if (!IsDraining) + MarkBodyInitiatedDrain(); + } + + // Accumulate each command's fresh error while draining, but do not duplicate an error + // already captured during its read phase or delivered to a live consumer. + var completeErrorIsOwnCancellation = completeError is { } completedWithError + && IsOwnCancellation(completedWithError.Error); + if ((consumeInternally || IsConsumingNonQuery || IsDraining && !_isResultReady) + && !capturedThisCommand && completeError is { } error + && !completeErrorIsOwnCancellation) + (_drainErrors ??= new()).Add(PgErrorException.Create(error.Error)); + + // Extended-query errors discard every following command through the next Sync. Skip + // those commands locally and consume the RFQ which is their only wire response. + if (completeError is not { TransactionStatus: TransactionStatus.Unknown }) + return; + + while (++_commandIndex < CommandCount && !_commands[_commandIndex].WithSync) { } + + if (IsAsync) + await ReadRfqAsync(_decoder!).ConfigureAwait(false); + else + ReadRfq(_decoder!); + + // Reaching the end means the discarded segment terminated at our appended Sync. + if (_commandIndex == CommandCount) + _readFlowRfq = false; + } + + void SubmitPublication(Context context, DetachedPublication publication) + { + Debug.Assert(_detachedPublication is DetachedPublication.None); + _detachedPublication = publication; + context.SubmitDetached((IThreadPoolWorkItem)this); + } + + private protected override void ExecuteDetachedWorkItem() + { + var publication = _detachedPublication; + _detachedPublication = DetachedPublication.None; + switch (publication) + { + case DetachedPublication.Result: + TrySetEnumeratorResult(true, runContinuationsAsynchronously: false); + break; + case DetachedPublication.Completion: + CompleteEnumeration(runContinuationsAsynchronously: false); + break; + default: + ThrowHelper.ThrowInvalidOperation("The command flow has no publication pending."); + break; + } + } + + void SetCallerCancellationToken(CancellationToken token) + { + var cancellation = GetOrCreateCancellationState(); + lock (cancellation) + { + cancellation.CallerToken = token; + RegisterCancellationCallbacksLocked(cancellation); + } + } + + void RegisterCancellationCallbacks(CancellationState cancellation) + { + lock (cancellation) + RegisterCancellationCallbacksLocked(cancellation); + } + + void RegisterCancellationCallbacksLocked(CancellationState cancellation) + { + if (cancellation.CallerToken.CanBeCanceled) + { + Debug.Assert(IsAsync); + if (cancellation.CallerRegistration == default) + cancellation.CallerRegistration = cancellation.CallerToken.UnsafeRegister(static (state, token) + => ((LegacyCommandFlow)state!).RequestCancelAndWake(token, CancellationScope.CurrentWindow), this); + } + if (cancellation.FlowToken.CanBeCanceled && cancellation.FlowRegistration == default) + { + cancellation.FlowRegistration = cancellation.FlowToken.UnsafeRegister(static (state, token) + => ((LegacyCommandFlow)state!).RequestCancelAndWake(token, CancellationScope.RemainingFlow), this); + } + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] + async ValueTask DisposeCancellationRegistrations(CancellationState cancellation) + { + CancellationTokenRegistration callerRegistration; + CancellationTokenRegistration flowRegistration; + lock (cancellation) + { + callerRegistration = cancellation.CallerRegistration; + cancellation.CallerRegistration = default; + flowRegistration = cancellation.FlowRegistration; + cancellation.FlowRegistration = default; + } + await callerRegistration.DisposeAsync().ConfigureAwait(false); + await flowRegistration.DisposeAsync().ConfigureAwait(false); + } + + bool IsCancellationToken(CancellationToken token) + { + var cancellation = Volatile.Read(ref _cancellationState); + return cancellation is not null + && (token == cancellation.CallerToken || token == cancellation.FlowToken); + } + + // Cancellation callbacks only latch intent and wake the body. The body delivers cancellation after + // it has restored the wire boundary and is ready to release execution tenure. + internal Task CancelAsync() + { + var delivery = GetOrCreateCancelDelivery(); + if (IsCompleted) + { + delivery.TrySetResult(); + return delivery.Task; + } + RequestCancelAndWake(default, CancellationScope.RemainingFlow); + if (IsCompleted) + delivery.TrySetResult(); + return delivery.Task; + } + + CancellationState GetOrCreateCancellationState() + { + if (Volatile.Read(ref _cancellationState) is { } cancellation) + return cancellation; + var created = new CancellationState(); + return Interlocked.CompareExchange(ref _cancellationState, created, null) ?? created; + } + + TaskCompletionSource GetOrCreateCancelDelivery() + { + var cancellation = GetOrCreateCancellationState(); + var delivery = Volatile.Read(ref cancellation.Delivery); + if (delivery is not null) + return delivery; + var created = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + return Interlocked.CompareExchange(ref cancellation.Delivery, created, null) ?? created; + } + + bool RequestCancel(CancellationToken token, CancellationScope scope, + BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, + BackendCancellationTiming subsequentTiming = BackendCancellationTiming.AfterGrace, + bool allowCompletedEnumeration = false) + { + if (IsEnumerationCompleted && !allowCompletedEnumeration) + return false; + var cancellation = GetOrCreateCancellationState(); + cancellation.DeliverToken = token; + var observedScope = Volatile.Read(ref cancellation.Scope); + while ((int)scope > observedScope) + { + var priorScope = Interlocked.CompareExchange(ref cancellation.Scope, (int)scope, observedScope); + if (priorScope == observedScope) + break; + observedScope = priorScope; + } + Volatile.Write(ref cancellation.Requested, true); + Volatile.Write(ref _draining, true); + var observedTiming = Volatile.Read(ref cancellation.Timing); + while ((int)timing > observedTiming) + { + var priorTiming = Interlocked.CompareExchange(ref cancellation.Timing, (int)timing, observedTiming); + if (priorTiming == observedTiming) + break; + observedTiming = priorTiming; + } + var observedSubsequentTiming = Volatile.Read(ref cancellation.SubsequentTiming); + while ((int)subsequentTiming > observedSubsequentTiming) + { + var priorTiming = Interlocked.CompareExchange(ref cancellation.SubsequentTiming, + (int)subsequentTiming, observedSubsequentTiming); + if (priorTiming == observedSubsequentTiming) + break; + observedSubsequentTiming = priorTiming; + } + var delivery = Volatile.Read(ref cancellation.Delivery); + RequestBackendCancellation(timing, delivery); + return true; + } + + void RequestCancelAndWake(CancellationToken token, CancellationScope scope) + { + if (!RequestCancel(token, scope)) + return; + var delivery = Volatile.Read(ref _cancellationState) is { } cancellation + ? Volatile.Read(ref cancellation.Delivery) + : null; + _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); + _callerInteractionCore.WakeBody(useDedicatedDriver: !IsAsync && delivery is not null); + } + + void RequestBackendCancellation(BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, + TaskCompletionSource? delivery = null) + { + // Cancellation which wins before body entry remains latched in the cancellation state and is + // replayed immediately after the execution context is published. + if (Volatile.Read(ref _contextPublished) && Volatile.Read(ref _cancellationState) is { } cancellation) + { + var key = Volatile.Read(ref cancellation.EpisodeKey); + if (key is null) + { + var created = new object(); + key = Interlocked.CompareExchange(ref cancellation.EpisodeKey, created, null) ?? created; + } + _context.RequestBackendCancellation(this, CancellationWindow, timing, delivery, + key, Volatile.Read(ref cancellation.Scope), + (BackendCancellationTiming)Volatile.Read(ref cancellation.SubsequentTiming)); + } + } + + bool IsOwnCancellation(PgError error) + { + if (Volatile.Read(ref _cancellationState) is not { } cancellation + || !Volatile.Read(ref cancellation.Requested) || error.SqlState != PgErrorCodes.QueryCanceled) + return false; + // PgDecoder records each ErrorResponse arrival exactly once. Classification may inspect the + // preserved error through several command-result paths, so it must not report the same strike. + return true; + } + + void EnterStoppingDrainIfNeeded(Context context) + { + if (_callerInteractionCore.CloseException is { } close && context.StoppingToken.IsCancellationRequested + && !IsDraining && !IsEnumerationCompleted) + { + CompleteEnumerationWithException(close); + MarkBodyInitiatedDrain(); + } + } + + // Publish progress unconditionally. A terminal delivery can lose its task-source CAS to an already- + // completed generation; a synchronous disposer must still observe this sticky level rather than park. + void SignalPumpProgress() + => _callerInteractionCore.SignalProgress(); + + void CompleteEnumerationWithException(Exception ex) + { + // Close state must survive task-source rearming, including flows whose body never started. + if (ex is PgClientClosedException or PgCollateralException) + _callerInteractionCore.SetCloseLatch(ex); + if (IsEnumerationCompleted) + return; + // Teardown may race the consumer. The task source is the completion authority; + // _enumeratorCompleted follows only when this call wins the current generation. + if (TrySetEnumeratorException(ex, runContinuationsAsynchronously: true)) + PublishEnumerationCompleted(); + // A faulted body will not publish another continuation. + SignalPumpProgress(); + // Wire recovery remains with the body or the framework recovery flow; this method only completes + // the consumer-facing generation. + } + + void PublishBodyTerminated() + { + Volatile.Write(ref _bodyState, BodyTerminated); + SignalPumpProgress(); + } + + bool TerminateBodyBeforeStart() + => Interlocked.CompareExchange(ref _bodyState, BodyTerminated, BodyNotStarted) == BodyNotStarted; + + bool IsBodyRunning => Volatile.Read(ref _bodyState) == BodyRunning; + bool IsBodyTerminated => Volatile.Read(ref _bodyState) == BodyTerminated; + + // Source handoff finishes before body/consumer rendezvous begins, so both reuse the same wait event. + private protected override FlowHandoffEvent? HandoffEvent => _callerInteractionCore.GetWaitEvent(); + + // Return the rendezvous directly. An async wrapper could signal the disposer before registering the + // body's continuation, causing a late ThreadPool dispatch instead of caller-thread drain execution. + FlowCallerInteractionCore.CallerHandoffAwaitable YieldToCaller() + { + FieldRef> fieldRef; + unsafe + { + fieldRef = FieldRef>.Create(&GetCallerInteractionCore, this); + } + return _callerInteractionCore.YieldToCaller(fieldRef); + } + + static ref FlowCallerInteractionCore GetCallerInteractionCore(LegacyCommandFlow instance) + => ref instance._callerInteractionCore; + + protected override void OnAbort(Exception exception) => FaultCaller(exception); + + // Graceful stopping is the early wire-close wake and is idempotent across heartbeat ticks. + protected override void OnStopping(Exception exception) + { + if (!IsBodyRunning || !IsAsync) + { + FaultCaller(exception); + return; + } + + // Resume normally so the body observes the close latch and drains; abort faults the gate. + _callerInteractionCore.SetCloseLatch(exception); + _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); + } + + // Wake a running body so it owns fault delivery; directly fault a flow whose body never started. + void FaultCaller(Exception exception) + { + if (TerminateBodyBeforeStart()) + { + CompleteEnumerationWithException(exception); + // A synchronous flow may already have entered ExecuteAfterHandoff while its inner read + // body is still NotStarted. Terminating that body does not complete the outer execution: + // fault and wake its initial handoff so the framework task can settle and release tenure. + _callerInteractionCore.FaultBodyWait(exception); + _callerInteractionCore.WakeBody(); + return; + } + + // A concurrent body start may have beaten the pre-start terminal claim. + if (IsBodyRunning) + _callerInteractionCore.FaultBodyWait(exception); + else + CompleteEnumerationWithException(exception); + } + + internal override void Fail(Exception exception) => FaultCaller(exception); + + protected override void OnReleasing(Exception? exception) + { + if (Volatile.Read(ref _cancellationState) is { } cancellation) + Volatile.Read(ref cancellation.Delivery)?.TrySetResult(); + _commands.Return(); + } + + protected override void OnDiscarded() + { + // Discarded flows never enter the base release path. + GetObserver(out var observerState)?.OnCompleting(this, null, observerState); + _commands.Return(); + } + + protected override void OnReset() + { + Debug.Assert(IsPending || IsCompleted); + _commandIndex = -1; + _executePipelinedCore.Reset(); + ResetEnumeratorMoveNextSource(); + _enumeratorCurrent = default; + _enumeratorCompleted = false; + _isResultReady = false; + _callerInteractionCore.Reset(); + if (_cancellationState is { } cancellation) + { + cancellation.Reset(); + _cancellationState = null; + } + _drainErrors = null; + _consumeNonQuery = false; + _nonQueryRecordsAffected = 0; + _consumerDisposed = false; + _draining = false; + _drainModeEntered = false; + WaitForDrainOnDispose = true; + // Dispatch state is per-tenure. + _contextPublished = false; + _context = default; + _bodyState = BodyNotStarted; + _consumerAdvanced = false; + } + + FlowCallerInteractionCoreResult IValueTaskSource.GetResult(short token) + => _callerInteractionCore.ConsumeGateResult(token); + + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) + => _callerInteractionCore.GateStatus(token); + + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + { + _callerInteractionCore.OnGateCompleted(continuation, state, token, flags); + // Drain is a sticky level. Recheck it after registering so an earlier gate edge cannot be lost + // across gate reset. This is the body's suspending stack, so the wake must dispatch asynchronously. + if (IsDraining) + { + // A synchronous takeover would already have resumed the body inline. Reaching this callback + // means autonomous execution still owns the body. + IsAsync = true; + _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); + } + } + + // Backing for the pipelined-dispatch ValueTask. Returned to the framework when activation + // hasn't fired yet. Nested callback completes it when ExecutePipelined finishes. + void IValueTaskSource.GetResult(short token) => _executePipelinedCore.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _executePipelinedCore.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _executePipelinedCore.OnCompleted(continuation, state, token, flags); + +} diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 28e0c0f..5f6d5ec 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -579,7 +579,7 @@ internal void RequestBackendCancellation(PgClientFlow instigator, int window, internal void RequestBackendCancellation(PgClientFlow instigator, int window, BackendCancellationTiming timing, TaskCompletionSource? delivery = null) => RequestBackendCancellation(instigator, window, timing, delivery, new object(), - (int)Flows.CommandFlow.CancellationScope.CurrentWindow, timing); + (int)Flows.LegacyCommandFlow.CancellationScope.CurrentWindow, timing); /// Returns an awaitable for the decoder. Activation is a cross-flow rendezvous completed by /// another flow's thread, so GetResult throws if not yet completed - async bodies await, diff --git a/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs b/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs index 19395c4..fcd99c7 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs @@ -55,7 +55,7 @@ internal void RequestServerCancellation(PgClientFlow instigator, coordinator = GetOrCreateCancellationCoordinatorLocked(); } coordinator.RequestCancellation(instigator, window, timing, delivery, - episodeKey, scope == (int)Flows.CommandFlow.CancellationScope.RemainingFlow, + episodeKey, scope == (int)Flows.LegacyCommandFlow.CancellationScope.RemainingFlow, subsequentTiming); // Flow release publishes its coordinator down-edge before IsCompleted. Recheck after // publishing the episode so either that down-edge or this observation retires it. From 4bdf524efa389d99e44419ed58b92a098122bbc3 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 01:42:56 +0200 Subject: [PATCH 090/136] Detach cancellation bookkeeping from the legacy flow --- Slon/Pg/Protocol/PgClientFlow.cs | 2 +- Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 5f6d5ec..5a163b8 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -579,7 +579,7 @@ internal void RequestBackendCancellation(PgClientFlow instigator, int window, internal void RequestBackendCancellation(PgClientFlow instigator, int window, BackendCancellationTiming timing, TaskCompletionSource? delivery = null) => RequestBackendCancellation(instigator, window, timing, delivery, new object(), - (int)Flows.LegacyCommandFlow.CancellationScope.CurrentWindow, timing); + (int)Flows.CommandExecutionCancellationScope.CurrentWindow, timing); /// Returns an awaitable for the decoder. Activation is a cross-flow rendezvous completed by /// another flow's thread, so GetResult throws if not yet completed - async bodies await, diff --git a/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs b/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs index fcd99c7..0902d5a 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.Cancellation.cs @@ -55,7 +55,7 @@ internal void RequestServerCancellation(PgClientFlow instigator, coordinator = GetOrCreateCancellationCoordinatorLocked(); } coordinator.RequestCancellation(instigator, window, timing, delivery, - episodeKey, scope == (int)Flows.LegacyCommandFlow.CancellationScope.RemainingFlow, + episodeKey, scope == (int)Flows.CommandExecutionCancellationScope.RemainingFlow, subsequentTiming); // Flow release publishes its coordinator down-edge before IsCompleted. Recheck after // publishing the episode so either that down-edge or this observation retires it. From 54ba4dcd843c63f3bd181a90b4f8bec55eb1881e Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 11:49:42 +0200 Subject: [PATCH 091/136] Fast-path single-command execution internally --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 72e4a7c..385f7f5 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -257,6 +257,9 @@ readonly struct CommandExecutionCore(TOps ops) readonly TOps _ops = ops; ref CommandExecutionState _state => ref _ops.State; internal bool IsResultReady => Volatile.Read(ref _state.Phase) is PhaseResultReady; + bool IsSinglePublishedCommand + => _state.Commands.Count is 1 + && !_state.Commands.ItemRef(0).SuppressEnumeration; [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] @@ -453,7 +456,9 @@ bool First() WaitForReadySynchronously(); Debug.Assert(!_state.ConsumerDetached); RegisterCancellation(default); - var result = ReadNextPublishedResult(); + var result = IsSinglePublishedCommand + ? ReadResult(0) + : ReadNextPublishedResult(); return result is not null && PublishSynchronousResult(result); } catch (TimeoutException ex) @@ -488,6 +493,12 @@ bool NextBatch() SkipDiscardedCommands(); _state.CommandIndex++; + if (IsSinglePublishedCommand) + { + CompleteBatch(); + _state.ConsumerObservedCompletion = true; + return false; + } var next = ReadNextPublishedResult(); if (next is not null) return PublishSynchronousResult(next); @@ -595,7 +606,9 @@ async ValueTask FirstAsync(CancellationToken cancellationToken) await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); Debug.Assert(!_state.ConsumerDetached); RegisterCancellation(cancellationToken); - var result = await ReadNextPublishedResultAsync().ConfigureAwait(false); + var result = IsSinglePublishedCommand + ? await ReadResultAsync(0).ConfigureAwait(false) + : await ReadNextPublishedResultAsync().ConfigureAwait(false); if (result is null) return false; _state.Current = result; @@ -660,6 +673,12 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) await SkipDiscardedCommandsAsync().ConfigureAwait(false); _state.CommandIndex++; + if (IsSinglePublishedCommand) + { + await CompleteBatchAsync().ConfigureAwait(false); + _state.ConsumerObservedCompletion = true; + return false; + } var next = await ReadNextPublishedResultAsync().ConfigureAwait(false); if (next is not null) { From 763d9917a6d03641bda9e4ce0079a224b38ad490 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 12:02:07 +0200 Subject: [PATCH 092/136] Remove the legacy command flow --- Slon.Tests/Ado/FlowMigrationTests.cs | 6 +- Slon.Tests/CommandFlowImplementation.cs | 6 - Slon.Tests/FlowBindingProbe.cs | 30 +- Slon.Tests/Pg/CommandDrainTests.cs | 6 - .../Pg/CommandResultEnumerationTests.cs | 14 - Slon.Tests/Pg/CommandUserCancellationTests.cs | 4 - Slon.Tests/Pg/ExclusiveAccessFlowTests.cs | 6 +- .../Pg/PublicCommandFlowSurfaceTests.cs | 1 - Slon.Tests/Pg/RacingDisposeInMemoryTests.cs | 20 - Slon.Tests/Slon.Tests.csproj | 2 - .../Flows/LegacyCommandFlow.Enumerator.cs | 446 ------ Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs | 1331 ----------------- test-command-flows.sh | 9 - 13 files changed, 28 insertions(+), 1853 deletions(-) delete mode 100644 Slon.Tests/CommandFlowImplementation.cs delete mode 100644 Slon/Pg/Protocol/Flows/LegacyCommandFlow.Enumerator.cs delete mode 100644 Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs delete mode 100755 test-command-flows.sh diff --git a/Slon.Tests/Ado/FlowMigrationTests.cs b/Slon.Tests/Ado/FlowMigrationTests.cs index 0f6da67..fe32258 100644 --- a/Slon.Tests/Ado/FlowMigrationTests.cs +++ b/Slon.Tests/Ado/FlowMigrationTests.cs @@ -184,10 +184,6 @@ static async Task DrainAsync(CommandFlow flow) } static async Task DrainAsync(BindingProbeFlow flow) - { - var e = flow.GetAsyncEnumerator(); - while (await e.MoveNextAsync()) { } - await e.DisposeAsync(); - } + => await flow.WaitForComplete(); } diff --git a/Slon.Tests/CommandFlowImplementation.cs b/Slon.Tests/CommandFlowImplementation.cs deleted file mode 100644 index ac92053..0000000 --- a/Slon.Tests/CommandFlowImplementation.cs +++ /dev/null @@ -1,6 +0,0 @@ -#if !COMMAND_FLOW_NEXT -global using CommandFlow = Slon.Pg.Protocol.Flows.LegacyCommandFlow; -global using CommandFlowObserver = Slon.Pg.Protocol.Flows.LegacyCommandFlowObserver; -global using CommandFlowOptions = Slon.Pg.Protocol.Flows.LegacyCommandFlowOptions; -#endif -global using ReplacementCommandFlow = Slon.Pg.Protocol.Flows.CommandFlow; diff --git a/Slon.Tests/FlowBindingProbe.cs b/Slon.Tests/FlowBindingProbe.cs index e655860..bb9db11 100644 --- a/Slon.Tests/FlowBindingProbe.cs +++ b/Slon.Tests/FlowBindingProbe.cs @@ -1,7 +1,6 @@ using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; using Slon.Pg; -using LegacyCommandFlow = Slon.Pg.Protocol.Flows.LegacyCommandFlow; namespace Slon.Tests; @@ -10,17 +9,40 @@ sealed class BindingProbeContext(string name) : PgClientFlowBindingContext internal string Name { get; } = name; } -sealed class BindingProbeFlow(bool fail = false) : LegacyCommandFlow(async: true, []) +sealed class BindingProbeFlow : PgClientFlow { + readonly bool _fail; internal int BindCount { get; private set; } internal string? ContextName { get; private set; } + internal BindingProbeFlow(bool fail = false) + : base(supportsDeferredFlush: true) + { + _fail = fail; + IsAsync = true; + } + + protected override bool EnableActivationTimeout => true; + + protected override ValueTask ExecuteAuto(Context context) + { + var write = new CommandList(Command.Create("select 1")) + .WriteCommandsAsync(context.GetEncoder(), appendSync: true); + return new(new FlowTasks(write, DrainAsync(context))); + + static async ValueTask DrainAsync(Context context) + { + var decoder = await context.GetDecoderAsync().ConfigureAwait(false); + while (context.OutstandingRfqCount is not 0) + _ = await decoder.GetNextAsync().ConfigureAwait(false); + } + } + internal override void Bind(PgClientFlowBindingContext? context) { BindCount++; ContextName = ((BindingProbeContext)context!).Name; - if (fail) + if (_fail) throw new InvalidOperationException("binding rejected"); - Initialize(IsAsync, Command.Create("select 1")); } } diff --git a/Slon.Tests/Pg/CommandDrainTests.cs b/Slon.Tests/Pg/CommandDrainTests.cs index b899f8c..1db2d09 100644 --- a/Slon.Tests/Pg/CommandDrainTests.cs +++ b/Slon.Tests/Pg/CommandDrainTests.cs @@ -226,9 +226,7 @@ public async Task ConsumerDispose_MidBatch_BodyDrainsRemaining_ConnectionUsable( // wire usable (a hang shows as the "select 1" WaitAsync timing out, not a suite hang). [TestMethod] [DoNotParallelize] -#if COMMAND_FLOW_NEXT [Ignore("Exercises the legacy body coroutine's open-before-park rendezvous.")] -#endif public async Task ConsumerDispose_MidBatch_SyncDispose_OpenBeforePark_Stress() { var iters = StressEnv.Iterations(fallback: 8, cap: 8_000); @@ -253,9 +251,7 @@ public async Task ConsumerDispose_MidBatch_SyncDispose_OpenBeforePark_Stress() // CompleteEnumeration, or CompleteEnumerationWithException), completing the body cross-thread while the pump is parked - the // sticky terminal publication must wake it. No other live-server test reaches this interleaving. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises the legacy body coroutine's in-flight completion/pump handoff race.")] -#endif public async Task InFlightCompletion_RacesSyncDispose_PumpNeverStrands_Stress() { // Each iteration is a full connect + force-abort cycle. Cap it because this is path coverage, @@ -345,9 +341,7 @@ public async Task StoppingToken_MidBatch_FollowedByDispose_FlowCompletesCleanly( // gate await is faulted by the heartbeat-driven OnStopping and the consumer's MoveNext // surfaces PgClientClosedException without any delivery. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Requires the legacy body coroutine to read and publish a result before any consumer advances the flow.")] -#endif public async Task StoppingToken_PreFireAsync_BodyFaultsWithoutDelivery() { var protocol = await PgTestPool.NewIsolatedAsync(); diff --git a/Slon.Tests/Pg/CommandResultEnumerationTests.cs b/Slon.Tests/Pg/CommandResultEnumerationTests.cs index 39e54a3..2aa8c31 100644 --- a/Slon.Tests/Pg/CommandResultEnumerationTests.cs +++ b/Slon.Tests/Pg/CommandResultEnumerationTests.cs @@ -128,9 +128,7 @@ public async Task DescribeOnlyErrorSurfacesWhenInspectingTheResult() } [ConnectionCreatingTestMethod] -#if COMMAND_FLOW_NEXT [Ignore("The replacement fixes the execution/consumption mode for a flow tenure; switching an async flow to synchronous driving is a legacy body-rendezvous behavior.")] -#endif public async Task AsyncFlow_CanSwitchToSynchronousResultAdvancement() { await using var protocol = await PgTestPool.NewIsolatedAsync(); @@ -244,12 +242,8 @@ public async Task ErrorWithoutSync_ResumesAfterInternalSync(bool async) public async Task Reset_ClearsEnumerationCompleted_ForNextTenure() { await using var protocol = await PgTestPool.NewIsolatedAsync(); -#if COMMAND_FLOW_NEXT var flow = new CommandFlow( async: true, enableActivationTimeout: false, Command.Create("select 1")); -#else - var flow = new ResettableCommandFlow(async: true, Command.Create("select 1")); -#endif for (var tenure = 0; tenure < 2; tenure++) { if (tenure > 0) @@ -266,12 +260,4 @@ public async Task Reset_ClearsEnumerationCompleted_ForNextTenure() } } - // Pooling a timeout-armed flow is refused by Reset. Opt out so the reset path itself is testable. -#if !COMMAND_FLOW_NEXT - sealed class ResettableCommandFlow(bool async, params ReadOnlySpan commands) - : CommandFlow(async, commands) - { - protected override bool EnableActivationTimeout => false; - } -#endif } diff --git a/Slon.Tests/Pg/CommandUserCancellationTests.cs b/Slon.Tests/Pg/CommandUserCancellationTests.cs index cf5c7ca..5eaf2fd 100644 --- a/Slon.Tests/Pg/CommandUserCancellationTests.cs +++ b/Slon.Tests/Pg/CommandUserCancellationTests.cs @@ -131,9 +131,7 @@ public async Task UserCt_FiresMidRead_SurfacesOce_ProtocolUsable() // Autonomous execution may enter the command read before the consumer supplies its per-read // token. The late token must still arm backend cancellation for that active read. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Requires the legacy body to enter a read before the consumer supplies its token; the replacement consumer owns the read from entry.")] -#endif public async Task UserCt_SuppliedAfterReadStarted_RequestsCancellation_ProtocolUsable() { await using var blocker = await PgAdvisoryLock.AcquireAsync(); @@ -574,9 +572,7 @@ public async Task ConsumerDispose_UsesItsOwnGraceBeforeSideChannelAttempt() } [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Requires a second body-owned drain read after the consumer read times out; the replacement retains one read owner through drain.")] -#endif public async Task ServerCancel_ReadTimeoutAfterAmbiguousRetryAbortsWire() { var iterations = StressEnv.Iterations(fallback: 1, cap: 5_000); diff --git a/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs b/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs index e4797c6..23dfdd6 100644 --- a/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs +++ b/Slon.Tests/Pg/ExclusiveAccessFlowTests.cs @@ -38,11 +38,7 @@ static async Task DrainAsync(CommandFlow flow) } static async Task DrainBindingProbeAsync(BindingProbeFlow flow) - { - var e = flow.GetAsyncEnumerator(); - while (await e.MoveNextAsync()) { } - await e.DisposeAsync(); - } + => await flow.WaitForComplete(); [TestMethod] public async Task Scope_RoundTrip_RunsCommandOnInnerPipeline() diff --git a/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs b/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs index f878597..ca21bd4 100644 --- a/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs +++ b/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs @@ -24,6 +24,5 @@ public void ReplacementIsTheSealedPublicCommandFlowSurface() Assert.IsNotNull(flow.GetConstructor([ typeof(bool), typeof(Slon.Pg.Protocol.Flows.CommandFlowOptions).MakeByRefType() ])); - Assert.IsFalse(typeof(LegacyCommandFlow).IsPublic); } } diff --git a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs index de43791..5e1e3b6 100644 --- a/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs +++ b/Slon.Tests/Pg/RacingDisposeInMemoryTests.cs @@ -399,9 +399,7 @@ public Task RunLoop() => Task.Run(async () => // a forceful abort faults that read -> body's closed catch. Pre-fix: rethrow escaped DisposeAsync. // Now: DisposeAsyncCore swallows it. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises legacy body-driven throw and caller-gate ordering.")] -#endif public async Task Ordering1_BodyDrivenThrow_DisposeConverges() { await using var s = await BuildToFirstResultParked(); @@ -424,9 +422,7 @@ public async Task Ordering1_BodyDrivenThrow_DisposeConverges() // This isolates terminal publication and the continuation handoff without ThreadPool admission, // PostgreSQL timing, or an advisory lock. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises takeover of the legacy body coroutine during synchronous disposal.")] -#endif public async Task SyncDispose_InFlightReadFault_Converges() { var iterations = Math.Clamp(StressEnv.Iterations(fallback: 1, cap: int.MaxValue), 1, 500); @@ -455,9 +451,7 @@ public async Task SyncDispose_InFlightReadFault_Converges() // bounded exactly as production bounds it - advance past CompletionTimeout (30s) so the graceful->abort // escalation faults the parked drain read and DisposeAsync converges, swallowing the close. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises the legacy caller gate winning before the body throw.")] -#endif public async Task Ordering2_GateFirstThrow_DisposeConverges() { await using var s = await BuildToFirstResultParked(); @@ -482,9 +476,7 @@ public async Task Ordering2_GateFirstThrow_DisposeConverges() // consume that progress without receiving a continuation and transfer to AwaitDrainOnDispose. // Escalation must still redrive/fault the held read and complete both teardown participants. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises the legacy caller gate's progress-before-takeover ordering.")] -#endif public async Task SyncDispose_GateProgressBeforeTakeover_Converges() { var iterations = Math.Clamp(StressEnv.Iterations(fallback: 1, cap: int.MaxValue), 1, 500); @@ -512,9 +504,7 @@ public async Task SyncDispose_GateProgressBeforeTakeover_Converges() // progress-only wake; only afterwards does releasing the response let the body publish its sync // handoff continuation. Null progress cannot be mistaken for body termination. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises late publication of a legacy body handoff continuation.")] -#endif public async Task SyncDispose_ProgressWakeBeforeLateHandoff_DrivesBodyToTermination() { var clock = new FakeTimeProvider(); @@ -554,9 +544,7 @@ public async Task SyncDispose_ProgressWakeBeforeLateHandoff_DrivesBodyToTerminat // A sync-at-bind body parked between command results still owns a continuation after close becomes // consumer-terminal. Dispose must drive that continuation before returning or surfacing the close. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises the legacy body's inter-result caller gate.")] -#endif public async Task SyncFlow_CloseAtInterResultPark_DisposeRetainsDriveObligation() { var clock = new FakeTimeProvider(); @@ -601,9 +589,7 @@ public async Task SyncFlow_CloseAtInterResultPark_DisposeRetainsDriveObligation( // published CancelException, so the consumer's next MoveNextAsync self-delivers the close and the // loop converges. Asserts convergence (a hang here would regress that protection). [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises self-delivery after a no-op fault on the legacy caller gate.")] -#endif public async Task Ordering3_GateFaultNoOp_SelfDeliverConverges() { await using var s = await BuildToFirstResultParked(); @@ -628,9 +614,7 @@ public async Task Ordering3_GateFaultNoOp_SelfDeliverConverges() // abort lands CompleteEnumerationWithException on the LIVE (Reset) generation, because every body read in this flow // is preceded by a consumer Reset. Documents why the read-fault alone cannot produce the hang. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises the legacy body's read-fault-to-caller-gate transition.")] -#endif public async Task Ordering3_ReadFaultPath_NeverNoOps_Converges() { await using var s = await BuildToFirstResultParked(); @@ -656,9 +640,7 @@ public async Task Ordering3_ReadFaultPath_NeverNoOps_Converges() // inline ping-pong), so this is a convergence regression test, not an isolation of any one waker. The // never-started lost-completion (the actual reported hang) is gated by the stress repro + dump. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises graceful close while the legacy body is parked at its inter-result gate.")] -#endif public async Task MultiCommand_GracefulCloseAtInterResultGate_Converges() { await using var s = await BuildMultiToFirstResultParked(); @@ -691,9 +673,7 @@ public async Task MultiCommand_GracefulCloseAtInterResultGate_Converges() // the stale result 1. Ordering3_GateFaultNoOp asserts the read loop converges; a stale re-yield does // not hang, so this pins the specific next-call outcome the loop cannot catch. [TestMethod] -#if COMMAND_FLOW_NEXT [Ignore("Exercises stale continuation suppression in the legacy caller gate.")] -#endif public async Task GateFaultBeforeNextMoveNext_SelfDeliversClose_NeverReYieldsStale() { await using var s = await BuildToFirstResultParked(); diff --git a/Slon.Tests/Slon.Tests.csproj b/Slon.Tests/Slon.Tests.csproj index 1df14ac..5273122 100644 --- a/Slon.Tests/Slon.Tests.csproj +++ b/Slon.Tests/Slon.Tests.csproj @@ -8,8 +8,6 @@ false true $(NoWarn);SLONPG001;SLONPOOL001 - Legacy - $(DefineConstants);COMMAND_FLOW_NEXT diff --git a/Slon/Pg/Protocol/Flows/LegacyCommandFlow.Enumerator.cs b/Slon/Pg/Protocol/Flows/LegacyCommandFlow.Enumerator.cs deleted file mode 100644 index fc07e38..0000000 --- a/Slon/Pg/Protocol/Flows/LegacyCommandFlow.Enumerator.cs +++ /dev/null @@ -1,446 +0,0 @@ -using System.Collections; -using System.Threading.Tasks.Sources; - -namespace Slon.Pg.Protocol.Flows; - -partial class LegacyCommandFlow -{ - Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _enumeratorMoveNextTaskSource; - int _enumeratorMoveNextCompletionClaim; - // Serializes move-next rearming against body termination. Otherwise Reset can replace the generation - // just before terminal completion and strand the consumer (see MoveNextRearm.tla). Never hold it while - // dispatching the gate, which may run the body inline. - Slon.Threading.SpinLock _rearmLock; - CommandResult? _enumeratorCurrent; - bool _enumeratorCompleted; - bool _isResultReady; - // Consumer-thread-only. The first call uses the initial source generation; later calls rearm it. - // Body start is not a substitute because an executor-driven body may finish before first consumption. - bool _consumerAdvanced; - - bool IsEnumerationCompleted => Volatile.Read(ref _enumeratorCompleted); - void PublishEnumerationCompleted() => Volatile.Write(ref _enumeratorCompleted, true); - - ValueTask EnumeratorMoveNextTask => new(this, _enumeratorMoveNextTaskSource.Version); - - bool IValueTaskSource.GetResult(short token) => _enumeratorMoveNextTaskSource.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _enumeratorMoveNextTaskSource.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - => _enumeratorMoveNextTaskSource.OnCompleted(continuation, state, token, flags); - - bool TrySetEnumeratorResult(bool result, bool runContinuationsAsynchronously) - { - if (Interlocked.CompareExchange(ref _enumeratorMoveNextCompletionClaim, 1, 0) != 0) - return false; - _enumeratorMoveNextTaskSource.SetResult(result, runContinuationsAsynchronously); - return true; - } - - bool TrySetEnumeratorException(Exception exception, bool runContinuationsAsynchronously) - { - if (Interlocked.CompareExchange(ref _enumeratorMoveNextCompletionClaim, 1, 0) != 0) - return false; - _enumeratorMoveNextTaskSource.SetException(exception, runContinuationsAsynchronously); - return true; - } - - void ResetEnumeratorMoveNextSource() - { - _enumeratorMoveNextTaskSource.Reset(); - Volatile.Write(ref _enumeratorMoveNextCompletionClaim, 0); - } - - // Consumer completion must dispatch asynchronously because it may run while the pipeline still owns - // the current execution frame. - void CompleteEnumeration(bool runContinuationsAsynchronously = true) - { - // Drain errors outrank cancellation and clean completion. Preserve every error across a batch. - if (_drainErrors is { Count: > 0 } errors) - { - Exception fault = errors.Count == 1 ? errors[0] : new AggregateException(errors); - TrySetEnumeratorException(fault, runContinuationsAsynchronously); - } - else if (Volatile.Read(ref _cancellationState) is { DeliverOce: true } cancellation - && !_consumerDisposed) - TrySetEnumeratorException( - new OperationCanceledException(cancellation.DeliverToken), runContinuationsAsynchronously); - else - TrySetEnumeratorResult(false, runContinuationsAsynchronously); - // _enumeratorCompleted was set by the caller (SetResult's completed branch) before this runs. - SignalPumpProgress(); - } - - // Cancellation stops waiting; it does not stop the autonomous drain or escape from disposal. - async ValueTask AwaitDrainOnDispose() - { - var cancellationToken = Volatile.Read(ref _cancellationState)?.FlowToken ?? default; - try - { - await WaitForComplete(cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - // Caller cancelled the wait; unwind. The body drains autonomously in the background. - } - catch (PgClientClosedException) - { - // The flow completed via a protocol close; that is a clean terminal for a disposing consumer. - return; - } - // Flow completion is independent of errors accumulated while draining. - if (_drainErrors is { Count: > 0 } errors) - throw errors.Count == 1 ? errors[0] : new AggregateException(errors); - } - - void AwaitDrainOnDisposeSynchronously() - { - var cancellationToken = Volatile.Read(ref _cancellationState)?.FlowToken ?? default; - try - { - WaitForCompleteSynchronously(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - // Caller cancelled the wait; the body continues draining autonomously. - } - catch (PgClientClosedException) - { - return; - } - - if (_drainErrors is { Count: > 0 } errors) - throw errors.Count == 1 ? errors[0] : new AggregateException(errors); - } - - void CompleteEnumerationWithClose(Exception closeException) - { - // A result may already own this generation. Preserve the close in the latch, but only publish - // terminal enumeration state when the close wins the generation; otherwise the consumer must - // observe the result once, rearm, and self-deliver the latched close on its next move. - _callerInteractionCore.SetCloseLatch(closeException); - if (TrySetEnumeratorException(closeException, runContinuationsAsynchronously: true)) - PublishEnumerationCompleted(); - SignalPumpProgress(); - } - - // Consumer terminality can precede body terminality under shutdown. Transfer a live body to the - // autonomous driver without re-entering MoveNext: its gate may already be faulted while its handoff - // continuation is pending. - bool TransferLiveBodyToDrain() - { - if (IsBodyTerminated) - return false; - MarkTerminalConsumerGone(); - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); - _callerInteractionCore.WakeBody(); - return true; - } - - // Terminal state outlives task-source generations. Close takes precedence over caller cancellation; - // absent either, a terminal enumeration completes cleanly. - void EnsureEnumerationCompleted() - { - if (_enumeratorMoveNextTaskSource.GetStatus(_enumeratorMoveNextTaskSource.Version) is not ValueTaskSourceStatus.Pending) - return; - if (_callerInteractionCore.CloseException is { } latched) - TrySetEnumeratorException(latched, runContinuationsAsynchronously: true); - else - { - var cancellation = Volatile.Read(ref _cancellationState); - var effectiveCancellationToken = EffectiveCancellationToken; - if (cancellation is { } && Volatile.Read(ref cancellation.Requested) - || effectiveCancellationToken.IsCancellationRequested) - TrySetEnumeratorException( - new OperationCanceledException(effectiveCancellationToken.IsCancellationRequested - ? effectiveCancellationToken - : cancellation!.DeliverToken), - runContinuationsAsynchronously: true); - // Rearming after a clean terminal still needs to complete the new generation. - else if (IsEnumerationCompleted) - TrySetEnumeratorResult(false, runContinuationsAsynchronously: true); - } - } - - public readonly struct Enumerator(LegacyCommandFlow flow) : IEnumerator, IAsyncEnumerator - { - // Here so we can pass the cancellation token and enumerate without boxing the struct (which WithCancellation must do). - /// - public Enumerator GetAsyncEnumerator() => this; - - // Dispose always calls MoveNext to confirm the enumerator is done without tracking additional state. - // So this method should be resilient to multiple fetches of *at least* the final result. - /// - public bool MoveNext() - { - if (flow is null) - return false; - - // Queueing only established FIFO position. This is the first point at which the caller - // is ready to take the source pump and drive the synchronous body. - if (!flow._consumerAdvanced && !flow.IsAsyncAtDispatch) - flow.WaitForSyncHandoff(); - - var takeOverAsyncGate = false; - - // Guard-decide-rearm serialized against the body's terminal (see _rearmLock). The using scope - // ends before the WaitForContinuation drive below (which runs the body inline and would - // re-enter this non-reentrant lock); try/finally release keeps it lock-safe on any throw. - using (flow._rearmLock.EnterScope()) - { - // See MoveNextAsync: terminal enumeration state can outlive a completed source generation. - // Ensure the current generation carries that terminal before returning it. - if (flow.IsEnumerationCompleted) - { - flow.EnsureEnumerationCompleted(); - return flow.EnumeratorMoveNextTask.Result; - } - - if (flow.IsAsync) - { - if (flow._enumeratorCurrent is null) - ThrowHelper.ThrowInvalidOperation("No immediate sync/async mixing is allowed, the first MoveNext{Async} call has to match the async argument passed during initialize."); - flow.IsAsync = false; - takeOverAsyncGate = true; - } - - // See MoveNextAsync: rearm only on a non-first call; the first-call source is fresh and the - // body's first delivery lands on it. - if (flow._consumerAdvanced) - flow.ResetEnumeratorMoveNextSource(); - flow._consumerAdvanced = true; - } - // Close-latch self-deliver (sync): under close this call completes the generation it just - // armed, on its own thread. - if (flow._callerInteractionCore.CloseException is { } syncClosed) - { - flow.CompleteEnumerationWithClose(syncClosed); - return flow.EnumeratorMoveNextTask.Result; - } - // The body may already be parked on the async inter-result gate. Once this caller changes - // the flow to synchronous driving, open that gate inline so the body can hand its continuation - // to the rendezvous below. The edge also covers an in-flight body that has not parked yet. - if (takeOverAsyncGate) - { - flow._callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); - var delivered = flow.EnumeratorMoveNextTask; - if (delivered.IsCompleted) - return delivered.Result; - } - // A progress wake may precede both result completion and publication of the body's next - // handoff continuation (close faults the gate before a resumed body reaches YieldToCaller). - // Keep rendezvousing until either the result owns this turn or there is body work to drive. - while (true) - { - var continuation = flow._callerInteractionCore.WaitForContinuation(); - var task = flow.EnumeratorMoveNextTask; - if (task.IsCompleted) - { - if (continuation is not null) - flow._callerInteractionCore.DeferContinuation(continuation); - return task.Result; - } - continuation ??= flow._callerInteractionCore.TryTakePendingContinuation(); - if (continuation is null) - continue; - continuation.Invoke(); - } - } - - // DisposeAsync always calls MoveNextAsync to confirm the enumerator is done without tracking additional state. - // So this method should be resilient to multiple fetches of the final result. - /// - public ValueTask MoveNextAsync() => MoveNextAsync(default); - - /// Advances the enumerator asynchronously to the next element of the collection. - /// A that may be used to cancel the asynchronous operation. - /// A that will complete with a result of if the enumerator was successfully advanced to the next element, or if the enumerator has passed the end of the collection. - public ValueTask MoveNextAsync(CancellationToken cancellationToken) - { - if (flow is null) - return new(false); - - if (cancellationToken.IsCancellationRequested) - { - flow.GetOrCreateCancellationState().CallerToken = cancellationToken; - if (flow.RequestCancel(cancellationToken, CancellationScope.CurrentWindow)) - { - flow._callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); - flow._callerInteractionCore.WakeBody(); - } - return ValueTask.FromException(new OperationCanceledException(cancellationToken)); - } - - // The guard-decide-rearm is serialized against the body's terminal (see _rearmLock): the using - // scope covers the _enumeratorCompleted read through the move-next Reset, and ends before the - // gate dispatch below (which runs the body inline and would re-enter this non-reentrant lock). - // try/finally release keeps it lock-safe on any throw. - using (flow._rearmLock.EnterScope()) - { - // Publish the per-read token before the terminal repair below: a flow may already have - // completed before its first consumer call, and EnsureEnumerationCompleted needs this token to - // distinguish a pre-fired cancellation from a clean end. - if (cancellationToken.CanBeCanceled) - { - flow.SetCallerCancellationToken(cancellationToken); - } - - // Terminal enumeration state may outlive its completed source generation. Complete the - // newly armed generation with the same close, cancellation, or clean-end outcome. - if (flow.IsEnumerationCompleted) - { - flow.EnsureEnumerationCompleted(); - return flow.EnumeratorMoveNextTask; - } - - if (!flow.IsAsync) - { - if (flow._enumeratorCurrent is null) - ThrowHelper.ThrowInvalidOperation("No immediate sync/async mixing is allowed, the first MoveNext{Async} call has to match the async argument passed during initialize."); - flow.IsAsync = true; - } - - // The first delivery targets the initial generation. After consumer disposal, keep the - // current generation for the body's one-shot terminal. Body-initiated drain retains a live - // consumer and therefore continues rearming. - if (flow._consumerAdvanced && !Volatile.Read(ref flow._consumerDisposed)) - flow.ResetEnumeratorMoveNextSource(); - flow._consumerAdvanced = true; - } - // Drive the body; teardown may already have faulted the gate. - flow._callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); - // Read the close latch after Reset and complete the generation just armed. - if (flow._callerInteractionCore.CloseException is { } closed) - flow.CompleteEnumerationWithClose(closed); - return flow.EnumeratorMoveNextTask; - } - - public CommandResult Current => flow?._enumeratorCurrent ?? default!; - - /// - public void Dispose() - { - if (flow is null) - return; - - // Consumer terminality can precede body terminality under shutdown. A terminal consumer must - // still transfer a live body to autonomous drain, but must not re-enter the ordinary MoveNext - // pump: its gate may already be faulted while its handoff continuation is pending. - if (flow.IsEnumerationCompleted) - { - FinishCompletedDisposal(flow); - return; - } - - // The final result's terminal message has already been consumed. Only the outer - // enumeration's false publication remains, so finish through the ordinary consumer - // path rather than reclassifying structural terminality as autonomous abandonment. - if (flow.IsFullyConsumedFinalResult) - { - if (MoveNext()) - ThrowHelper.ThrowInvalidOperation( - "A fully consumed physical final result produced another result."); - FinishCompletedDisposal(flow); - return; - } - - // Synchronous disposal takes over an async body through a two-way rendezvous. A gate-parked - // body resumes inline; an in-flight body later hands over its continuation. Waiting on this - // rendezvous rather than MoveNext avoids blocking on the task the body itself must complete. - if (flow.IsAsync) - { - flow.IsAsync = false; - if (flow.WaitForDrainOnDispose) - flow.MarkConsumerWaitForDrain(); - else - flow.MarkConsumerGone(); - // Resume a gate-parked body inline; otherwise buffer the edge. - flow._callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); - if (flow.WaitForDrainOnDispose) - { - DriveBodyToTermination(flow); - // Drain ran on this thread; surface accumulated drain errors (completes immediately). - flow.AwaitDrainOnDisposeSynchronously(); - } - else - { - // Fast-return: wake the body to drain autonomously in the background, then return. - flow._callerInteractionCore.WakeBody(); - } - return; - } - - // Cancellation graduates a synchronous body to the dedicated driver. The disposer no - // longer pumps the same body concurrently; it only waits for tenure release when requested. - flow.MarkSyncConsumerGone(); - if (flow.WaitForDrainOnDispose) - flow.AwaitDrainOnDisposeSynchronously(); - - static void DriveBodyToTermination(LegacyCommandFlow flow) - { - while (!flow.IsBodyTerminated) - { - var continuation = flow._callerInteractionCore.WaitForContinuation(); - if (continuation is null) - { - if (flow.IsBodyTerminated) - break; - continuation = flow._callerInteractionCore.TryTakePendingContinuation(); - if (continuation is null) - continue; - } - continuation.Invoke(); - } - } - - static void FinishCompletedDisposal(LegacyCommandFlow flow) - { - if (flow.TransferLiveBodyToDrain() && flow.WaitForDrainOnDispose) - flow.AwaitDrainOnDisposeSynchronously(); - } - } - - /// - public ValueTask DisposeAsync() - { - if (flow is null) - return new(); - - if (flow.IsEnumerationCompleted) - return FinishCompletedDisposalAsync(flow); - - if (flow.IsFullyConsumedFinalResult) - return FinishFinalResultAsync(this, flow); - - // Mark autonomous drain, open the async result gate, and wake any synchronous rendezvous. - // Pipeline tenure keeps successors behind the body until it reaches RFQ. - if (flow.WaitForDrainOnDispose) flow.MarkConsumerWaitForDrain(); else flow.MarkConsumerGone(); - flow._callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); - flow._callerInteractionCore.WakeBody(); - // Optionally await the body's bounded completion; cancellation stops waiting, not draining. - if (flow.WaitForDrainOnDispose) - return flow.AwaitDrainOnDispose(); - return new(); - - static async ValueTask FinishFinalResultAsync(Enumerator enumerator, LegacyCommandFlow flow) - { - if (await enumerator.MoveNextAsync().ConfigureAwait(false)) - ThrowHelper.ThrowInvalidOperation( - "A fully consumed physical final result produced another result."); - await FinishCompletedDisposalAsync(flow).ConfigureAwait(false); - } - - static ValueTask FinishCompletedDisposalAsync(LegacyCommandFlow flow) - { - // A completed awaited drain may still have errors to surface. - flow.TransferLiveBodyToDrain(); - return flow.WaitForDrainOnDispose ? flow.AwaitDrainOnDispose() : new(); - } - } - - /// - void IEnumerator.Reset() => throw new NotSupportedException(); - /// - object? IEnumerator.Current => Current; - } - -} diff --git a/Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs b/Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs deleted file mode 100644 index 33948de..0000000 --- a/Slon/Pg/Protocol/Flows/LegacyCommandFlow.cs +++ /dev/null @@ -1,1331 +0,0 @@ -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; -using System.Threading.Tasks.Sources; -using Slon.Runtime.CompilerServices; -// A unique result type distinguishes the caller gate from this flow's other IValueTaskSource instantiations. -using FlowCallerInteractionCoreResult = System.ValueTuple; - -namespace Slon.Pg.Protocol.Flows; - -[Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] -internal abstract class LegacyCommandFlowObserver : PgClientFlowObserver -{ - protected internal virtual void OnStarted(LegacyCommandFlow flow, object? state) { } - protected internal virtual void OnCommandResult(LegacyCommandFlow flow, CommandResult result, object? state) { } - protected internal virtual void OnDrainStarted(LegacyCommandFlow flow, object? state) { } -} - -[Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] -internal readonly struct LegacyCommandFlowOptions -{ - public LegacyCommandFlowObserver? Observer { get; init; } - public object? ObserverState { get; init; } - public CommandList Commands { get; init; } - // Optional per-flow override for time spent waiting in the protocol backlog. - public TimeSpan? PendingTimeout { get; init; } -} - -[Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] -internal partial class LegacyCommandFlow : PgClientFlow, IValueTaskSource, IValueTaskSource, IValueTaskSource -{ - sealed class PreparationReadState - { - internal ParameterTypeList ParameterTypes; - } - - static readonly TimeSpan ConsumerDrainCancellationGracePeriod = TimeSpan.FromSeconds(1); - - internal override bool DefersSyncHandoff => true; - - internal enum CancellationScope : byte - { - None, - CurrentWindow = 1, - RemainingFlow = 2 - } - - // Flow state - CommandList _commands; - TimeSpan? _pendingTimeout; - FlowCallerInteractionCore _callerInteractionCore; - // Cancellation is cold; keep its tokens, registrations and attribution state off ordinary flows. - CancellationState? _cancellationState; - // Errors encountered while draining are surfaced by a waiting DisposeAsync. Live consumers observe - // their errors directly, and the list remains unallocated on the successful path. - List? _drainErrors; - // Set only by ConsumeNonQueryAsync, after enqueue but before any consumer-side gate release. - // The body cannot reach first publication until such a release, and every release publishes - // this write, so plain accesses suffice and the body observes the mode at first wake. - bool _consumeNonQuery; - bool IsConsumingNonQuery => _consumeNonQuery; - bool IsConsumingAutonomously => IsDraining || IsConsumingNonQuery; - long _nonQueryRecordsAffected; - - // Once draining, the body bypasses result handoffs and reads autonomously to RFQ. This is state, not - // an I/O cancellation token: canceling the I/O would prevent restoration of a clean wire boundary. - bool _draining; - internal bool IsDraining => Volatile.Read(ref _draining); - // Body-thread-only guard: later commands must not change the drive mode chosen on drain entry. - bool _drainModeEntered; - - - // Distinguishes consumer disposal from a body-initiated drain; disposal suppresses terminal OCE delivery. - bool _consumerDisposed; - // When true, DisposeAsync awaits the body's drain to RFQ. Otherwise it returns while the body drains; - // pipeline retirement still prevents the next flow from observing a dirty wire. - internal bool WaitForDrainOnDispose { get; set; } = true; - - // Consumer disposal without waiting for the autonomous drain. - void MarkConsumerGone() - { - Volatile.Write(ref _consumerDisposed, true); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, - BackendCancellationTiming.AtReadFrontier); - } - - // Consumer disposal while waiting for the autonomous drain. - void MarkConsumerWaitForDrain() - { - Volatile.Write(ref _consumerDisposed, true); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, - BackendCancellationTiming.AtReadFrontier); - } - - // A synchronous consumer cannot abandon its drive obligation while the body is parked. Give the - // cancellation path a delivery source so WakeBody transfers that obligation to the dedicated driver. - void MarkSyncConsumerGone() - { - Volatile.Write(ref _consumerDisposed, true); - _ = GetOrCreateCancelDelivery(); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.AfterGrace, - BackendCancellationTiming.AtReadFrontier); - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); - _callerInteractionCore.WakeBody(useDedicatedDriver: true); - } - - // Enumeration already ended; only transfer the live body tail to autonomous ownership. - // There is no unfinished consumer work to justify a backend cancellation intent. - void MarkTerminalConsumerGone() - { - Volatile.Write(ref _consumerDisposed, true); - Volatile.Write(ref _draining, true); - } - - // Result publication orders these body-owned fields before the consumer can observe the - // CommandResult. IsComplete is consumer-owned after that handoff. A behavior-limited reader may - // consider its visible result final while later commands still exist, so use the physical - // command index rather than an ADO-visible result count. - bool IsFullyConsumedFinalResult - => _isResultReady && _commandIndex >= CommandCount - 1 - && _enumeratorCurrent is { IsComplete: true }; - - // A body-initiated drain keeps the consumer attached for terminal cancellation or close delivery. - void MarkBodyInitiatedDrain() => Volatile.Write(ref _draining, true); - - // Result-production state - RowDescription? _requestedRowDescription; - PgError? _pgError; - int _commandIndex = -1; - PgDecoder? _decoder; - bool _readFlowRfq; - - // Pipelined dispatch state. Lives here (not on PgClientFlow base) because the shared-promise - // optimization that needs these fields is LegacyCommandFlow-specific (see DispatchPipelinedRead). - Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore _executePipelinedCore; - Context _context; - bool _contextPublished; - // Consumer terminality, body terminality, and framework release are distinct phases. Start and - // pre-start termination race during shutdown, so one atomic state owns that decision. - const int BodyNotStarted = 0; - const int BodyRunning = 1; - const int BodyTerminated = 2; - int _bodyState; - DetachedPublication _detachedPublication; - - enum DetachedPublication : byte - { - None, - Result, - Completion - } - - sealed class CancellationState - { - internal CancellationToken CallerToken; - internal CancellationToken FlowToken; - internal CancellationTokenRegistration CallerRegistration; - internal CancellationTokenRegistration FlowRegistration; - internal bool Requested; - internal int Scope; - internal int Timing; - internal int SubsequentTiming; - internal CancellationToken DeliverToken; - internal TaskCompletionSource? Delivery; - internal object? EpisodeKey; - internal bool DeliverOce; - - internal void Reset() - { - CallerToken = default; - FlowToken = default; - CallerRegistration.Dispose(); - CallerRegistration = default; - FlowRegistration.Dispose(); - FlowRegistration = default; - Requested = false; - Scope = (int)CancellationScope.None; - Timing = (int)BackendCancellationTiming.AfterGrace; - SubsequentTiming = (int)BackendCancellationTiming.AfterGrace; - DeliverToken = default; - Delivery = null; - EpisodeKey = null; - DeliverOce = false; - } - } - LegacyCommandFlow() : base(supportsDeferredFlush: true) - { - _callerInteractionCore.Initialize(); - } - - // Interactive commands carry caller patience, so arm the activation timeout. - protected override bool EnableActivationTimeout => true; - protected override TimeSpan? PendingTimeout => _pendingTimeout; - internal override TimeSpan? BackendCancellationGracePeriod - => Volatile.Read(ref _consumerDisposed) ? ConsumerDrainCancellationGracePeriod : null; - - internal LegacyCommandFlow(bool async, params ReadOnlySpan commands) : this() - => Initialize(async, commands); - internal LegacyCommandFlow(bool async, in LegacyCommandFlowOptions options) : this() - => Initialize(async, options); - - private protected LegacyCommandFlow(bool async, TimeSpan? pendingTimeout) : this() - { - IsAsync = async; - _pendingTimeout = pendingTimeout; - } - - internal LegacyCommandFlow Initialize(bool async, params ReadOnlySpan commands) - => Initialize(async, options: new() { Commands = new(commands) }); - - internal LegacyCommandFlow Initialize(bool async, in LegacyCommandFlowOptions options) - { - IsAsync = async; - if (options.Observer is { } observer) - SetObserver(observer, options.ObserverState); - _commands = options.Commands; - _pendingTimeout = options.PendingTimeout; - options.Observer?.OnStarted(this, options.ObserverState); - return this; - } - - // Declares internal non-query consumption and runs it to completion. The single entry point - // makes the ownership rule structural: no enumerator is exposed on this path, so mixing - // enumeration with internal consumption is unrepresentable. The declaration precedes any - // consumer-side gate release, so the body observes it at first wake and never publishes. - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - internal async ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) - { - _consumeNonQuery = true; - _nonQueryRecordsAffected = -1; - var enumerator = GetAsyncEnumerator(cancellationToken); - try - { - // Release a body parked pre-publication and wake the flow. - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: false); - _callerInteractionCore.WakeBody(); - _ = await EnumeratorMoveNextTask.ConfigureAwait(false); - // Internal consumption owns error delivery. Errors collected during the internal - // drain have no other outlet: DisposeAsync deliberately skips a completed flow. - if (_drainErrors is { Count: > 0 } errors) - throw errors.Count == 1 ? errors[0] : new AggregateException(errors); - return _nonQueryRecordsAffected; - } - finally - { - await enumerator.DisposeAsync().ConfigureAwait(false); - } - } - - // The token in force for the current read: the flow token once fired (whole-flow cancel), else the - // per-read token if cancelable, else the flow token. - CancellationToken EffectiveCancellationToken - => GetEffectiveCancellationToken(Volatile.Read(ref _cancellationState)); - - static CancellationToken GetEffectiveCancellationToken(CancellationState? cancellation) - => cancellation is null ? default - : cancellation.FlowToken.IsCancellationRequested ? cancellation.FlowToken - : cancellation.CallerToken.CanBeCanceled ? cancellation.CallerToken - : cancellation.FlowToken; - - public int CommandCount => _commands.Count; - internal virtual int VisibleCommandCount => _commands.VisibleCount; - public bool IsResultReady => _isResultReady; - - public Enumerator GetEnumerator() - { - return new Enumerator(this); - } - - // Bind at submission because eager writing precedes the first MoveNextAsync. - internal override void BindCallerToken(CancellationToken cancellationToken) - => GetOrCreateCancellationState().FlowToken = cancellationToken; - internal override CancellationToken MigrationCancellationToken - => Volatile.Read(ref _cancellationState)?.FlowToken ?? default; - - public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) - { - // A missing per-call token must not replace the flow token captured at submission. - if (cancellationToken.CanBeCanceled) - GetOrCreateCancellationState().FlowToken = cancellationToken; - return new(this); - } - - protected override ValueTask ExecuteAuto(Context context) - { - if (!IsAsync && _callerInteractionCore.IsWaiting) - return ExecuteAfterHandoff(context); - - return new(ExecuteAutoCore(context)); - } - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask ExecuteAfterHandoff(Context context) - { - try - { - await YieldToCaller(); - } - catch (Exception ex) - { - TerminateBodyBeforeStart(); - CompleteEnumerationWithException(ex); - throw; - } - - return ExecuteAutoCore(context); - } - - FlowTasks ExecuteAutoCore(Context context) - { - _context = context; - Volatile.Write(ref _contextPublished, true); - if (Volatile.Read(ref _cancellationState) is { } cancellation) - { - if (cancellation.FlowToken.IsCancellationRequested) - RequestCancel(cancellation.FlowToken, CancellationScope.RemainingFlow); - else if (Volatile.Read(ref cancellation.Requested)) - RequestBackendCancellation((BackendCancellationTiming)Volatile.Read(ref cancellation.Timing), - Volatile.Read(ref cancellation.Delivery)); - } - ValueTask writeTask; - try - { - // Writes are independent of consumer admission. Inter-result gates provide backpressure. - // Async writes use transport completion; sync writes use the resumable non-blocking path so - // the caller thread retains execution ownership across readiness waits. - var appendSync = !_commands[CommandCount - 1].WithSync; - _readFlowRfq = appendSync; - // Caller cancellation never cancels wire I/O. A partially cancelled write requires - // protocol recovery and can strand already-pipelined successors; the body instead - // observes the latched intent and drains every written command to RFQ. - writeTask = IsAsync - ? _commands.WriteCommandsAsync(context.GetEncoder(), appendSync, default) - : WriteCommandsResumable(context, appendSync); - - // Observe synchronous faults here; pending writes remain the framework-owned trailing task. - if (writeTask.IsCompleted) - writeTask.GetAwaiter().GetResult(); - } - catch (Exception ex) - { - TerminateBodyBeforeStart(); - CompleteEnumerationWithException(ex); - throw; - } - - // Read and write run concurrently; the framework observes the trailing write before releasing - // the flow, preserving single-writer tenure without blocking reads behind socket backpressure. - return new FlowTasks( - trailingExecutionTask: writeTask, - pipelineTask: DispatchPipelinedRead( - context, context.GetProtocolStatic().Promise)); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - ValueTask WriteCommandsResumable(Context context, bool appendSync) - { - var encoder = context.GetEncoder(); - ValueTask writeTask; - using (encoder.BeginResumableWriteScope()) - writeTask = _commands.WriteCommandsResumable(encoder, appendSync); - return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); - } - - // Defer state-machine creation until activation because all flows share one protocol-static promise. - ValueTask DispatchPipelinedRead(Context context, ValueTaskSourcePromise promise) - { - // The shared promise may be tenured only after successful decoder activation. - var waiter = context.GetDecoderAsync().ConfigureAwait(false); - if (waiter.IsCompleted) - { - // Only successful activation owns the shared promise. A settled fault belongs to this flow's - // private completion source because it never claimed the wire. - if (!waiter.IsCompletedSuccessfully) - { - // Preserve the activation's close, timeout, or cancellation identity. - try { waiter.GetAwaiter().GetResult(); } - catch (Exception ex) { _executePipelinedCore.SetException(ex); } - return new ValueTask(this, _executePipelinedCore.Version); - } - // Handing the shared-promise-backed task to the framework is safe: the contract guarantees - // the waiter is consumed (releasing the promise tenure via GetResult's Reset) before the - // item's position is republished, so a successor's dispatch always finds the tenure released. - PromiseAsyncValueTaskMethodBuilder.Promise = promise; - try - { - return ExecutePipelined(context); - } - finally - { - PromiseAsyncValueTaskMethodBuilder.Promise = null; - } - } - - // Static continuation: a bridge into framework state, so no captured scheduling context is needed. - waiter.OnCompleted(static state => - { - var flow = (LegacyCommandFlow)state!; - var ctx = flow._context; - // A faulted activation never claimed the shared promise; complete only this flow's source. - var activation = ctx.GetDecoderAsync().GetAwaiter(); - if (!activation.IsCompletedSuccessfully) - { - // Preserve the activation's close, timeout, or cancellation identity. - try { activation.GetResult(); } - catch (Exception ex) { flow._executePipelinedCore.SetException(ex); } - return; - } - var promise = ctx.GetProtocolStatic().Promise; - PromiseAsyncValueTaskMethodBuilder.Promise = promise; - ValueTask task = flow.ExecutePipelined(ctx); - try - { - if (!task.IsCompleted) - { - ((IValueTaskSource)promise).OnCompleted(static state => - { - var flow = (LegacyCommandFlow)state!; - try - { - var promise = flow._context - .GetProtocolStatic().Promise; - ((IValueTaskSource)promise).GetResult(promise.Token); - flow._executePipelinedCore.SetResult(true); - } - catch (Exception ex) - { - flow._executePipelinedCore.SetException(ex); - } - // This internal bridge runs no user code and requires no ExecutionContext flow. - }, flow, promise.Token, ValueTaskSourceOnCompletedFlags.None); - } - else - { - try - { - task.GetAwaiter().GetResult(); - flow._executePipelinedCore.SetResult(true); - } - catch (Exception ex) - { - flow._executePipelinedCore.SetException(ex); - } - } - } - finally - { - PromiseAsyncValueTaskMethodBuilder.Promise = null; - } - }, this); - - return new ValueTask(this, _executePipelinedCore.Version); - } - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder))] - async ValueTask ExecutePipelined(Context context) - { - // Pre-start teardown may have already claimed terminality. In that case the consumer has its - // fault and this late dispatch has no body tenure to establish. - if (Interlocked.CompareExchange(ref _bodyState, BodyRunning, BodyNotStarted) != BodyNotStarted) - return; - try - { - // If we have a continuation stored we must already be on the caller thread, - // otherwise we must make sure to unblock the executor (see comment in the write phase). - // This first handoff is body execution too: close may fault it, so it belongs inside the - // same terminal envelope that publishes the consumer fault and body termination. - if (!IsAsync && !_callerInteractionCore.HasHandoff) - await YieldToCaller(); - - // User cancellation must not cancel activation or wire I/O. The flow observes it after - // activation, drains itself to RFQ, then delivers OCE without invoking pipeline recovery. - _decoder = await context.GetDecoderAuto().ConfigureAwait(false); - var publishedResult = false; - while (++_commandIndex < CommandCount) - { - _isResultReady = false; - bool hasPreparedDescription; - bool suppressEnumeration; - bool describeForPreparation; - { - ref readonly var command = ref _commands.ItemRef(_commandIndex); - _decoder.UseReadTimeout(command.Timeout); - suppressEnumeration = command.SuppressEnumeration; - describeForPreparation = command.DescribeForPreparation; - hasPreparedDescription = command.Descriptor is { IsPrepared: true, PreparedRowDescription: not null } - && !command.DescribeOnly; - } - - // Registrations only latch and wake; terminal delivery remains body-owned. Dispose them - // before promise tenure ends so callbacks cannot reach the next flow. Do not rearm after - // consumer disposal, where a persistent cancellation could escape its intended wait. - if (Volatile.Read(ref _cancellationState) is { } cancellationAtReadStart && !IsDraining - && (cancellationAtReadStart.CallerToken.CanBeCanceled - || cancellationAtReadStart.FlowToken.CanBeCanceled)) - RegisterCancellationCallbacks(cancellationAtReadStart); - // After close, a fresh command must not consume bytes left by its predecessor. A draining - // flow may continue reading its own response to restore RFQ. - if (!IsDraining && context.IsProtocolClosed) - throw context.FlowTerminationException; - - PreparationReadState? preparationRead = null; - if (describeForPreparation) - { - preparationRead = new(); - await ReadPreparationDescription(context, preparationRead).ConfigureAwait(false); - } - else if (IsAsync && hasPreparedDescription) - { - // Prepared commands with a known description have the compact BindComplete -> - // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes - // this outer body rather than a nested parser coroutine; the second message normally - // comes from the same batch and is consumed synchronously. - if (!_decoder.TryMoveNext()) - { - if (!await _decoder.MoveNextAsync().ConfigureAwait(false)) - _decoder.ThrowUnexpectedEof(); - } - var message = _decoder.Current; - - if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) - { - _pgError = bindError; - _requestedRowDescription = null; - } - else - { - if (!_decoder.TryMoveNext()) - { - if (!await _decoder.MoveNextAsync().ConfigureAwait(false)) - _decoder.ThrowUnexpectedEof(); - } - message = _decoder.Current; - message.DebugEnsureExpected(PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); - _pgError = null; - _requestedRowDescription = null; - } - } - else if (IsAsync) - { - var rowDescription = context.GetProtocolStatic().RowDescription; - var read = _commands.ItemRef(_commandIndex).ReadUntilExecuteAsync(_decoder, rowDescription); - (_pgError, _requestedRowDescription) = await read.ConfigureAwait(false); - } - else - { - var rowDescription = context.GetProtocolStatic().RowDescription; - (_pgError, _requestedRowDescription) = _commands.ItemRef(_commandIndex) - .ReadUntilExecute(_decoder, rowDescription); - } - - // A draining consumer cannot observe CommandResult, so retain read errors for disposal. - var capturedThisCommand = false; - var readErrorIsOwnCancellation = _pgError is { } readError - && IsOwnCancellation(readError); - if (IsConsumingAutonomously && _pgError is { } readErrorToCapture - && !readErrorIsOwnCancellation) - { - (_drainErrors ??= new()).Add(PgErrorException.Create(readErrorToCapture)); - capturedThisCommand = true; - } - - // Await in-flight callbacks before releasing shared promise tenure. - var cancellation = Volatile.Read(ref _cancellationState); - if (cancellation is not null) - await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); - // Cancellation switches to autonomous drain; terminal delivery follows RFQ and tenure release. - var effectiveCancellationToken = GetEffectiveCancellationToken(cancellation); - if ((cancellation is { } && Volatile.Read(ref cancellation.Requested) - || effectiveCancellationToken.IsCancellationRequested) && !IsEnumerationCompleted) - { - cancellation ??= GetOrCreateCancellationState(); - if (effectiveCancellationToken.IsCancellationRequested) - cancellation.DeliverToken = effectiveCancellationToken; - cancellation.DeliverOce = true; - if (!IsDraining) - MarkBodyInitiatedDrain(); - } - - var result = InitializeResult( - context, preparationRead); - ((LegacyCommandFlowObserver?)GetObserver(out var observerState)) - ?.OnCommandResult(this, result, observerState); - - // Disposal drains without another result handoff. Graceful close instead faults the - // attached consumer, then uses the same autonomous drain. Command errors remain results. - if (context.StoppingToken.IsCancellationRequested && !IsDraining - && !IsEnumerationCompleted) - { - // Latch the close (a consumer that Resets past this point self-delivers it), wake a - // parked consumer, then drain. - var close = context.FlowTerminationException; - _callerInteractionCore.SetCloseLatch(close); - CompleteEnumerationWithClose(close); - MarkBodyInitiatedDrain(); - } - if (!IsDraining && !IsConsumingNonQuery && !suppressEnumeration) - { - // Eager async execution must wait for the consumer to arm generation zero before - // publishing its first result. Synchronous execution already runs on that caller. - if (!publishedResult && IsAsync) - { - await _callerInteractionCore.WaitForCaller(this).ConfigureAwait(false); - // The first consumer can arrive after the response prelude was already read - // and its registrations were disposed. Its token was armed by MoveNextAsync; - // the result has now won that race, so retire the late registration before - // publishing the result. - if (Volatile.Read(ref _cancellationState) is { } lateCancellation) - await DisposeCancellationRegistrations(lateCancellation).ConfigureAwait(false); - EnterStoppingDrainIfNeeded(context); - } - - if (!IsDraining && !IsConsumingNonQuery) - { - _isResultReady = true; - publishedResult = true; - // Result continuations run asynchronously so the body can reach the next gate - // before user code asks for the next result. Buffered batches then advance inline - // from MoveNextAsync instead of suspending one Task state machine per result. - PublishEnumeratorResult(context, result); - - if (!IsDraining && !IsConsumingNonQuery) - { - if (IsAsync) - { - await _callerInteractionCore.WaitForCaller(this).ConfigureAwait(false); - EnterStoppingDrainIfNeeded(context); - } - else - await YieldToCaller(); - - /* The next MoveNext or MoveNextAsync call resumes here. */ - } - } - } - else if (!_drainModeEntered && IsAsyncAtDispatch && !IsAsync) - { - // An async I/O wake raced a synchronous disposer before the body reached its handoff. - if (WaitForDrainOnDispose) - { - // Hand the continuation to the disposer, which waits on the rendezvous rather than - // this task and can therefore drive the remaining drain without sync-over-async. - await YieldToCaller(); - } - else - { - // No disposer is waiting to drive; retain asynchronous background draining. - IsAsync = IsAsyncAtDispatch; - } - } - // Preserve the drive mode chosen on first drain entry. - _drainModeEntered = _drainModeEntered || IsDraining; - - // Disposing the message enumerator completes the command and, in drain mode, consumes its - // remaining rows. Re-read the current execution mode after every resumption. - (PgError Error, TransactionStatus TransactionStatus)? completeError; - // Consumption mode may change while the body is suspended; use the current value. - if (IsConsumingNonQuery || suppressEnumeration) - { - await CompleteInternalConsumptionAsync( - result, suppressEnumeration, capturedThisCommand).ConfigureAwait(false); - continue; - } - else if (IsAsync) - { - var resultEnumerator = context.GetProtocolStatic().ResultMessageEnumerator; - await resultEnumerator.DisposeAsync().ConfigureAwait(false); - completeError = resultEnumerator.CompleteError; - } - else - { - var resultEnumerator = context.GetProtocolStatic().ResultMessageEnumerator; - resultEnumerator.Dispose(); - completeError = resultEnumerator.CompleteError; - } - - if (result.Error is not null || completeError is not null) - await HandleCommandErrorsAsync( - result, suppressEnumeration, consumeInternally: false, - capturedThisCommand, completeError).ConfigureAwait(false); - } - - // The framework observes trailing write failure before releasing this flow. - if (_readFlowRfq) - { - if (_decoder.TryMoveNext()) - { - var message = _decoder.Current; - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - else if (IsAsync) - { - await ReadRfqAsync(_decoder).ConfigureAwait(false); - } - else - { - ReadRfq(_decoder); - } - } - - PublishEnumeratorResult(context, null); - } - catch (PgClientClosedException) when (context.IsProtocolClosed) - { - // Scope to our own closure so a nested protocol's close doesn't get treated as ours. - // Latch the close so a consumer that Resets after this point self-delivers it. - _callerInteractionCore.SetCloseLatch(context.FlowTerminationException); - // A detached consumer treats close as drain completion; a live consumer observes the close. - if (IsDraining) - { - if (!IsEnumerationCompleted) - PublishEnumeratorResult(context, null); - return; - } - CompleteEnumerationWithException(context.FlowTerminationException); - throw; - } - catch (OperationCanceledException ex) when (IsCancellationToken(ex.CancellationToken)) - { - CompleteEnumerationWithException(ex); - throw; - } - catch (TimeoutException ex) - { - await HandleTimeoutAsync(context, ex).ConfigureAwait(false); - return; - } - catch (Exception ex) - { - CompleteEnumerationWithException(ex); - throw; - } - finally - { - // The body is the sole owner of protocol-static row metadata. Recovery consumes only - // decoder/wire state, so a faulted body can release oversized storage while recovery - // retains the failed flow's framework tenure. - ref readonly var readState = ref context.GetProtocolStatic(); - readState.Reset(); - PublishBodyTerminated(); - } - } - - void PublishEnumeratorResult(Context context, CommandResult? next) - { - var completed = next is null; - var publishAsync = IsAsync; - if (completed) - { - _enumeratorCurrent = null; - } - else - { - if (Volatile.Read(ref _cancellationState) is { } cancellation) - cancellation.CallerToken = default; - - if (!ReferenceEquals(_enumeratorCurrent, next)) - _enumeratorCurrent = next; - } - - // Close is durable across generations; complete the current one without publishing a result. - if (_callerInteractionCore.CloseException is not null) - { - CompleteEnumerationWithClose(_callerInteractionCore.CloseException); - return; - } - if (completed) - { - // Publish durable terminal state atomically with respect to consumer rearming. Async - // consumers complete from the protocol scheduler so they cannot reenter this lock or - // the pipeline frame that still owns the shared promise; sync consumers retain their - // caller-driven completion. - using (_rearmLock.EnterScope()) - { - PublishEnumerationCompleted(); - if (!publishAsync) - CompleteEnumeration(); - } - if (publishAsync) - SubmitPublication(context, DetachedPublication.Completion); - return; - } - if (publishAsync) - { - // Queue the publication itself so the body reaches its next caller gate before user code - // resumes. Routing through the protocol scheduler preserves that ordering without forcing - // every result continuation onto the ThreadPool. - SubmitPublication(context, DetachedPublication.Result); - } - else - TrySetEnumeratorResult(true, runContinuationsAsynchronously: true); - } - - async ValueTask HandleTimeoutAsync(Context context, TimeoutException exception) - { - CompleteEnumerationWithException(exception); - RequestCancel(default, CancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, - BackendCancellationTiming.AtReadFrontier, allowCompletedEnumeration: true); - if (context.IsProtocolClosed) - ExceptionDispatchInfo.Throw(exception); - - // The timeout is terminal for the consumer, not for the body. Keep ownership of the - // command sequence and drain every remaining RFQ window; reaching each RFQ requests - // cancellation for each following window through the cancellation coordinator. Recovery is - // reserved for a failure of this semantic drain, where only wire obligations remain. - if (Volatile.Read(ref _cancellationState) is { } cancellation) - await DisposeCancellationRegistrations(cancellation).ConfigureAwait(false); - ((LegacyCommandFlowObserver?)GetObserver(out var observerState)) - ?.OnDrainStarted(this, observerState); - try - { - while (context.OutstandingRfqCount != 0) - _ = await _decoder!.GetNextAuto().ConfigureAwait(false); - } - catch (TimeoutException) - { - // The semantic drain owns the same cancellation episode. A timeout here would otherwise - // leave the episode unaware that its first read-timeout escalation made no protocol progress. - RequestCancel(default, CancellationScope.RemainingFlow, - BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier, - allowCompletedEnumeration: true); - throw; - } - } - - static async ValueTask ReadRfqAsync(PgDecoder decoder) - { - var message = await decoder.GetNextAsync().ConfigureAwait(false); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - - static void ReadRfq(PgDecoder decoder) - { - var message = decoder.GetNext(); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - CommandResult InitializeResult( - Context context, PreparationReadState? preparationRead) - { - ref readonly var readState = ref context.GetProtocolStatic(); - readState.ResultMessageEnumerator.Initialize(_commands.ItemRef(_commandIndex), _decoder!); - var result = _enumeratorCurrent ?? readState.CommandResult; - - ref readonly var command = ref _commands.ItemRef(_commandIndex); - var descriptor = command.Descriptor; - // We were preparing and we have no error from parse, make a prepared descriptor. - if (!descriptor.IsPrepared && !descriptor.CommandName.IsDefault - && (_pgError is not { } err || !err.Expected.Contains(PgTypes.BackendType.ParseComplete))) - { - descriptor = CommandDescriptor.CreatePrepared( - descriptor.CommandName, - preparationRead?.ParameterTypes ?? descriptor.ParameterTypes, - _requestedRowDescription?.Preserve()); - } - result.Initialize(this, _commandIndex, descriptor, _requestedRowDescription, - !command.DescribeOnly, command.IsSimple(), _pgError); - return result; - } - - ValueTask ReadPreparationDescription(Context context, PreparationReadState state) - { - var rowDescription = context.GetProtocolStatic().RowDescription; - ref readonly var command = ref _commands.ItemRef(_commandIndex); - if (!IsAsync) - { - (_pgError, state.ParameterTypes, _requestedRowDescription) = - command.ReadPreparationDescription(_decoder!, rowDescription); - return default; - } - - var read = command.ReadPreparationDescriptionAsync(_decoder!, rowDescription); - if (!read.IsCompletedSuccessfully) - return AwaitRead(this, state, read); - (_pgError, state.ParameterTypes, _requestedRowDescription) = read.Result; - return default; - - static async ValueTask AwaitRead( - LegacyCommandFlow flow, PreparationReadState state, - ValueTask<(PgError?, ParameterTypeList, RowDescription?)> read) - { - (flow._pgError, state.ParameterTypes, flow._requestedRowDescription) = - await read.ConfigureAwait(false); - } - } - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] - async ValueTask CompleteInternalConsumptionAsync( - CommandResult result, bool suppressEnumeration, bool capturedThisCommand) - { - while (_decoder!.Current.Header.Type is PgTypes.BackendType.DataRow) - { - if (!_decoder.TryMoveNext()) - await _decoder.GetNextAsync().ConfigureAwait(false); - } - result.CompleteNonQuery(_decoder.Current); - var completeError = await _commands.ItemRef(_commandIndex) - .CompleteAsync(_decoder).ConfigureAwait(false); - if (_pgError is null && completeError is null) - { - var recordsAffected = result.GetCommandComplete().BatchRecordsAffected; - if (recordsAffected >= 0) - _nonQueryRecordsAffected = _nonQueryRecordsAffected < 0 - ? recordsAffected - : checked(_nonQueryRecordsAffected + recordsAffected); - } - - if (result.Error is not null || completeError is not null) - await HandleCommandErrorsAsync( - result, suppressEnumeration, consumeInternally: true, - capturedThisCommand, completeError).ConfigureAwait(false); - } - - async ValueTask HandleCommandErrorsAsync( - CommandResult result, bool suppressEnumeration, bool consumeInternally, - bool capturedThisCommand, - (PgError Error, TransactionStatus TransactionStatus)? completeError) - { - var resultErrorIsOwnCancellation = result.Error is { } resultError - && IsOwnCancellation(resultError); - if (suppressEnumeration && result.Error is { } suppressedError - && !resultErrorIsOwnCancellation) - { - (_drainErrors ??= new()).Add(PgErrorException.Create(suppressedError)); - capturedThisCommand = true; - if (!IsDraining) - MarkBodyInitiatedDrain(); - } - - // Accumulate each command's fresh error while draining, but do not duplicate an error - // already captured during its read phase or delivered to a live consumer. - var completeErrorIsOwnCancellation = completeError is { } completedWithError - && IsOwnCancellation(completedWithError.Error); - if ((consumeInternally || IsConsumingNonQuery || IsDraining && !_isResultReady) - && !capturedThisCommand && completeError is { } error - && !completeErrorIsOwnCancellation) - (_drainErrors ??= new()).Add(PgErrorException.Create(error.Error)); - - // Extended-query errors discard every following command through the next Sync. Skip - // those commands locally and consume the RFQ which is their only wire response. - if (completeError is not { TransactionStatus: TransactionStatus.Unknown }) - return; - - while (++_commandIndex < CommandCount && !_commands[_commandIndex].WithSync) { } - - if (IsAsync) - await ReadRfqAsync(_decoder!).ConfigureAwait(false); - else - ReadRfq(_decoder!); - - // Reaching the end means the discarded segment terminated at our appended Sync. - if (_commandIndex == CommandCount) - _readFlowRfq = false; - } - - void SubmitPublication(Context context, DetachedPublication publication) - { - Debug.Assert(_detachedPublication is DetachedPublication.None); - _detachedPublication = publication; - context.SubmitDetached((IThreadPoolWorkItem)this); - } - - private protected override void ExecuteDetachedWorkItem() - { - var publication = _detachedPublication; - _detachedPublication = DetachedPublication.None; - switch (publication) - { - case DetachedPublication.Result: - TrySetEnumeratorResult(true, runContinuationsAsynchronously: false); - break; - case DetachedPublication.Completion: - CompleteEnumeration(runContinuationsAsynchronously: false); - break; - default: - ThrowHelper.ThrowInvalidOperation("The command flow has no publication pending."); - break; - } - } - - void SetCallerCancellationToken(CancellationToken token) - { - var cancellation = GetOrCreateCancellationState(); - lock (cancellation) - { - cancellation.CallerToken = token; - RegisterCancellationCallbacksLocked(cancellation); - } - } - - void RegisterCancellationCallbacks(CancellationState cancellation) - { - lock (cancellation) - RegisterCancellationCallbacksLocked(cancellation); - } - - void RegisterCancellationCallbacksLocked(CancellationState cancellation) - { - if (cancellation.CallerToken.CanBeCanceled) - { - Debug.Assert(IsAsync); - if (cancellation.CallerRegistration == default) - cancellation.CallerRegistration = cancellation.CallerToken.UnsafeRegister(static (state, token) - => ((LegacyCommandFlow)state!).RequestCancelAndWake(token, CancellationScope.CurrentWindow), this); - } - if (cancellation.FlowToken.CanBeCanceled && cancellation.FlowRegistration == default) - { - cancellation.FlowRegistration = cancellation.FlowToken.UnsafeRegister(static (state, token) - => ((LegacyCommandFlow)state!).RequestCancelAndWake(token, CancellationScope.RemainingFlow), this); - } - } - - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] - async ValueTask DisposeCancellationRegistrations(CancellationState cancellation) - { - CancellationTokenRegistration callerRegistration; - CancellationTokenRegistration flowRegistration; - lock (cancellation) - { - callerRegistration = cancellation.CallerRegistration; - cancellation.CallerRegistration = default; - flowRegistration = cancellation.FlowRegistration; - cancellation.FlowRegistration = default; - } - await callerRegistration.DisposeAsync().ConfigureAwait(false); - await flowRegistration.DisposeAsync().ConfigureAwait(false); - } - - bool IsCancellationToken(CancellationToken token) - { - var cancellation = Volatile.Read(ref _cancellationState); - return cancellation is not null - && (token == cancellation.CallerToken || token == cancellation.FlowToken); - } - - // Cancellation callbacks only latch intent and wake the body. The body delivers cancellation after - // it has restored the wire boundary and is ready to release execution tenure. - internal Task CancelAsync() - { - var delivery = GetOrCreateCancelDelivery(); - if (IsCompleted) - { - delivery.TrySetResult(); - return delivery.Task; - } - RequestCancelAndWake(default, CancellationScope.RemainingFlow); - if (IsCompleted) - delivery.TrySetResult(); - return delivery.Task; - } - - CancellationState GetOrCreateCancellationState() - { - if (Volatile.Read(ref _cancellationState) is { } cancellation) - return cancellation; - var created = new CancellationState(); - return Interlocked.CompareExchange(ref _cancellationState, created, null) ?? created; - } - - TaskCompletionSource GetOrCreateCancelDelivery() - { - var cancellation = GetOrCreateCancellationState(); - var delivery = Volatile.Read(ref cancellation.Delivery); - if (delivery is not null) - return delivery; - var created = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - return Interlocked.CompareExchange(ref cancellation.Delivery, created, null) ?? created; - } - - bool RequestCancel(CancellationToken token, CancellationScope scope, - BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, - BackendCancellationTiming subsequentTiming = BackendCancellationTiming.AfterGrace, - bool allowCompletedEnumeration = false) - { - if (IsEnumerationCompleted && !allowCompletedEnumeration) - return false; - var cancellation = GetOrCreateCancellationState(); - cancellation.DeliverToken = token; - var observedScope = Volatile.Read(ref cancellation.Scope); - while ((int)scope > observedScope) - { - var priorScope = Interlocked.CompareExchange(ref cancellation.Scope, (int)scope, observedScope); - if (priorScope == observedScope) - break; - observedScope = priorScope; - } - Volatile.Write(ref cancellation.Requested, true); - Volatile.Write(ref _draining, true); - var observedTiming = Volatile.Read(ref cancellation.Timing); - while ((int)timing > observedTiming) - { - var priorTiming = Interlocked.CompareExchange(ref cancellation.Timing, (int)timing, observedTiming); - if (priorTiming == observedTiming) - break; - observedTiming = priorTiming; - } - var observedSubsequentTiming = Volatile.Read(ref cancellation.SubsequentTiming); - while ((int)subsequentTiming > observedSubsequentTiming) - { - var priorTiming = Interlocked.CompareExchange(ref cancellation.SubsequentTiming, - (int)subsequentTiming, observedSubsequentTiming); - if (priorTiming == observedSubsequentTiming) - break; - observedSubsequentTiming = priorTiming; - } - var delivery = Volatile.Read(ref cancellation.Delivery); - RequestBackendCancellation(timing, delivery); - return true; - } - - void RequestCancelAndWake(CancellationToken token, CancellationScope scope) - { - if (!RequestCancel(token, scope)) - return; - var delivery = Volatile.Read(ref _cancellationState) is { } cancellation - ? Volatile.Read(ref cancellation.Delivery) - : null; - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); - _callerInteractionCore.WakeBody(useDedicatedDriver: !IsAsync && delivery is not null); - } - - void RequestBackendCancellation(BackendCancellationTiming timing = BackendCancellationTiming.AfterGrace, - TaskCompletionSource? delivery = null) - { - // Cancellation which wins before body entry remains latched in the cancellation state and is - // replayed immediately after the execution context is published. - if (Volatile.Read(ref _contextPublished) && Volatile.Read(ref _cancellationState) is { } cancellation) - { - var key = Volatile.Read(ref cancellation.EpisodeKey); - if (key is null) - { - var created = new object(); - key = Interlocked.CompareExchange(ref cancellation.EpisodeKey, created, null) ?? created; - } - _context.RequestBackendCancellation(this, CancellationWindow, timing, delivery, - key, Volatile.Read(ref cancellation.Scope), - (BackendCancellationTiming)Volatile.Read(ref cancellation.SubsequentTiming)); - } - } - - bool IsOwnCancellation(PgError error) - { - if (Volatile.Read(ref _cancellationState) is not { } cancellation - || !Volatile.Read(ref cancellation.Requested) || error.SqlState != PgErrorCodes.QueryCanceled) - return false; - // PgDecoder records each ErrorResponse arrival exactly once. Classification may inspect the - // preserved error through several command-result paths, so it must not report the same strike. - return true; - } - - void EnterStoppingDrainIfNeeded(Context context) - { - if (_callerInteractionCore.CloseException is { } close && context.StoppingToken.IsCancellationRequested - && !IsDraining && !IsEnumerationCompleted) - { - CompleteEnumerationWithException(close); - MarkBodyInitiatedDrain(); - } - } - - // Publish progress unconditionally. A terminal delivery can lose its task-source CAS to an already- - // completed generation; a synchronous disposer must still observe this sticky level rather than park. - void SignalPumpProgress() - => _callerInteractionCore.SignalProgress(); - - void CompleteEnumerationWithException(Exception ex) - { - // Close state must survive task-source rearming, including flows whose body never started. - if (ex is PgClientClosedException or PgCollateralException) - _callerInteractionCore.SetCloseLatch(ex); - if (IsEnumerationCompleted) - return; - // Teardown may race the consumer. The task source is the completion authority; - // _enumeratorCompleted follows only when this call wins the current generation. - if (TrySetEnumeratorException(ex, runContinuationsAsynchronously: true)) - PublishEnumerationCompleted(); - // A faulted body will not publish another continuation. - SignalPumpProgress(); - // Wire recovery remains with the body or the framework recovery flow; this method only completes - // the consumer-facing generation. - } - - void PublishBodyTerminated() - { - Volatile.Write(ref _bodyState, BodyTerminated); - SignalPumpProgress(); - } - - bool TerminateBodyBeforeStart() - => Interlocked.CompareExchange(ref _bodyState, BodyTerminated, BodyNotStarted) == BodyNotStarted; - - bool IsBodyRunning => Volatile.Read(ref _bodyState) == BodyRunning; - bool IsBodyTerminated => Volatile.Read(ref _bodyState) == BodyTerminated; - - // Source handoff finishes before body/consumer rendezvous begins, so both reuse the same wait event. - private protected override FlowHandoffEvent? HandoffEvent => _callerInteractionCore.GetWaitEvent(); - - // Return the rendezvous directly. An async wrapper could signal the disposer before registering the - // body's continuation, causing a late ThreadPool dispatch instead of caller-thread drain execution. - FlowCallerInteractionCore.CallerHandoffAwaitable YieldToCaller() - { - FieldRef> fieldRef; - unsafe - { - fieldRef = FieldRef>.Create(&GetCallerInteractionCore, this); - } - return _callerInteractionCore.YieldToCaller(fieldRef); - } - - static ref FlowCallerInteractionCore GetCallerInteractionCore(LegacyCommandFlow instance) - => ref instance._callerInteractionCore; - - protected override void OnAbort(Exception exception) => FaultCaller(exception); - - // Graceful stopping is the early wire-close wake and is idempotent across heartbeat ticks. - protected override void OnStopping(Exception exception) - { - if (!IsBodyRunning || !IsAsync) - { - FaultCaller(exception); - return; - } - - // Resume normally so the body observes the close latch and drains; abort faults the gate. - _callerInteractionCore.SetCloseLatch(exception); - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); - } - - // Wake a running body so it owns fault delivery; directly fault a flow whose body never started. - void FaultCaller(Exception exception) - { - if (TerminateBodyBeforeStart()) - { - CompleteEnumerationWithException(exception); - // A synchronous flow may already have entered ExecuteAfterHandoff while its inner read - // body is still NotStarted. Terminating that body does not complete the outer execution: - // fault and wake its initial handoff so the framework task can settle and release tenure. - _callerInteractionCore.FaultBodyWait(exception); - _callerInteractionCore.WakeBody(); - return; - } - - // A concurrent body start may have beaten the pre-start terminal claim. - if (IsBodyRunning) - _callerInteractionCore.FaultBodyWait(exception); - else - CompleteEnumerationWithException(exception); - } - - internal override void Fail(Exception exception) => FaultCaller(exception); - - protected override void OnReleasing(Exception? exception) - { - if (Volatile.Read(ref _cancellationState) is { } cancellation) - Volatile.Read(ref cancellation.Delivery)?.TrySetResult(); - _commands.Return(); - } - - protected override void OnDiscarded() - { - // Discarded flows never enter the base release path. - GetObserver(out var observerState)?.OnCompleting(this, null, observerState); - _commands.Return(); - } - - protected override void OnReset() - { - Debug.Assert(IsPending || IsCompleted); - _commandIndex = -1; - _executePipelinedCore.Reset(); - ResetEnumeratorMoveNextSource(); - _enumeratorCurrent = default; - _enumeratorCompleted = false; - _isResultReady = false; - _callerInteractionCore.Reset(); - if (_cancellationState is { } cancellation) - { - cancellation.Reset(); - _cancellationState = null; - } - _drainErrors = null; - _consumeNonQuery = false; - _nonQueryRecordsAffected = 0; - _consumerDisposed = false; - _draining = false; - _drainModeEntered = false; - WaitForDrainOnDispose = true; - // Dispatch state is per-tenure. - _contextPublished = false; - _context = default; - _bodyState = BodyNotStarted; - _consumerAdvanced = false; - } - - FlowCallerInteractionCoreResult IValueTaskSource.GetResult(short token) - => _callerInteractionCore.ConsumeGateResult(token); - - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) - => _callerInteractionCore.GateStatus(token); - - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - { - _callerInteractionCore.OnGateCompleted(continuation, state, token, flags); - // Drain is a sticky level. Recheck it after registering so an earlier gate edge cannot be lost - // across gate reset. This is the body's suspending stack, so the wake must dispatch asynchronously. - if (IsDraining) - { - // A synchronous takeover would already have resumed the body inline. Reaching this callback - // means autonomous execution still owns the body. - IsAsync = true; - _callerInteractionCore.ResumeBody(runContinuationsAsynchronously: true); - } - } - - // Backing for the pipelined-dispatch ValueTask. Returned to the framework when activation - // hasn't fired yet. Nested callback completes it when ExecutePipelined finishes. - void IValueTaskSource.GetResult(short token) => _executePipelinedCore.GetResult(token); - ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _executePipelinedCore.GetStatus(token); - void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) - => _executePipelinedCore.OnCompleted(continuation, state, token, flags); - -} diff --git a/test-command-flows.sh b/test-command-flows.sh deleted file mode 100755 index 750b398..0000000 --- a/test-command-flows.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/zsh -set -eu - -repo=${0:A:h} - -dotnet test "$repo/Slon.Tests/Slon.Tests.csproj" -c Release \ - -p:CommandFlowImplementation=Legacy "$@" -dotnet test "$repo/Slon.Tests/Slon.Tests.csproj" -c Release \ - -p:CommandFlowImplementation=Next "$@" From 78d4e7b5c10fba40d157c3feeeb7d744e917af17 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 12:10:53 +0200 Subject: [PATCH 093/136] Restore the scheduler chaos soak helper --- eng/scheduler-chaos | 8 ++++ eng/scheduler-chaos.c | 99 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100755 eng/scheduler-chaos create mode 100644 eng/scheduler-chaos.c diff --git a/eng/scheduler-chaos b/eng/scheduler-chaos new file mode 100755 index 0000000..a557010 --- /dev/null +++ b/eng/scheduler-chaos @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +binary="${TMPDIR:-/tmp}/slon-scheduler-chaos" + +cc -O2 -pthread "$here/scheduler-chaos.c" -o "$binary" +exec "$binary" "$@" diff --git a/eng/scheduler-chaos.c b/eng/scheduler-chaos.c new file mode 100644 index 0000000..ace2c1a --- /dev/null +++ b/eng/scheduler-chaos.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +static _Atomic int stopping; + +static void stop(int signal) +{ + (void)signal; + atomic_store_explicit(&stopping, 1, memory_order_relaxed); +} + +static uint32_t next(uint32_t *state) +{ + uint32_t value = *state; + value ^= value << 13; + value ^= value >> 17; + value ^= value << 5; + return *state = value; +} + +static void pause_ns(long nanoseconds) +{ + struct timespec duration = { .tv_nsec = nanoseconds }; + nanosleep(&duration, NULL); +} + +static void *churn(void *argument) +{ + uint32_t state = (uint32_t)(uintptr_t)argument ^ (uint32_t)clock(); + volatile uint64_t work = state; + + while (!atomic_load_explicit(&stopping, memory_order_relaxed)) + { + switch (next(&state) % 10) + { + case 0: + case 1: + sched_yield(); + break; + case 2: + case 3: + case 4: + pause_ns(1000L * (50 + next(&state) % 1950)); + break; + default: + { + // Keep a couple of cores busy in aggregate. macOS affinity tags are + // task-local hints, so light pressure could simply be placed away + // from the independently running test process. + const uint32_t iterations = 1000 + next(&state) % 200000; + for (uint32_t i = 0; i < iterations; i++) + work = work * 6364136223846793005ULL + 1; + break; + } + } + } + + return (void *)(uintptr_t)work; +} + +int main(int argc, char **argv) +{ + long cores = sysconf(_SC_NPROCESSORS_ONLN); + int threads = argc > 1 ? atoi(argv[1]) : (int)cores; + if (threads < 1) + { + fprintf(stderr, "usage: %s [threads]\n", argv[0]); + return 2; + } + + signal(SIGINT, stop); + signal(SIGTERM, stop); + + pthread_t *workers = calloc((size_t)threads, sizeof(*workers)); + if (workers == NULL) + return 1; + + fprintf(stderr, "scheduler chaos: %d threads, distributed pressure; Ctrl-C to stop\n", threads); + for (int i = 0; i < threads; i++) + { + if (pthread_create(&workers[i], NULL, churn, (void *)(uintptr_t)(i + 1)) != 0) + { + atomic_store(&stopping, 1); + threads = i; + break; + } + } + + for (int i = 0; i < threads; i++) + pthread_join(workers[i], NULL); + free(workers); + return 0; +} From 3c7a6f1eb823984fe953597a969a9f338f556231 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 12:24:15 +0200 Subject: [PATCH 094/136] Preserve cancellation cadence in the shared command flow --- Slon.Tests/Pg/CommandUserCancellationTests.cs | 54 +++++++++++++++++++ Slon/Pg/Protocol/Flows/CommandFlow.cs | 20 ++++--- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/Slon.Tests/Pg/CommandUserCancellationTests.cs b/Slon.Tests/Pg/CommandUserCancellationTests.cs index 5eaf2fd..8f1c613 100644 --- a/Slon.Tests/Pg/CommandUserCancellationTests.cs +++ b/Slon.Tests/Pg/CommandUserCancellationTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Time.Testing; using Slon.Pg; using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; @@ -571,6 +572,59 @@ public async Task ConsumerDispose_UsesItsOwnGraceBeforeSideChannelAttempt() await PgTestPool.RunAsync(protocol, "select 1"); } + [TestMethod] + [DataRow(false, DisplayName = "explicit cancellation")] + [DataRow(true, DisplayName = "consumer disposal")] + public async Task ServerCancel_SubsequentWindowDispatchesWithoutHeartbeat(bool dispose) + { + var time = new FakeTimeProvider(); + await using var blocker = await PgAdvisoryLock.AcquireAsync(); + var settleAttempt = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + await using var protocol = await PgTestPool.NewIsolatedAsync(o => + { + o.TimeProvider = time; + o.HeartbeatInterval = TimeSpan.FromHours(1); + o.CancelRequestDelay = TimeSpan.Zero; + o.CancelSender = (_, _, _) => new(settleAttempt.Task); + }); + + var flow = new CommandFlow(async: true, + Command.Create("select 1") with { WithSync = true }, + blocker.WaitCommand); + Assert.IsTrue(protocol.TryQueue(flow)); + var enumerator = flow.GetAsyncEnumerator(); + Assert.IsTrue(await enumerator.MoveNextAsync()); + + var cancellation = dispose + ? enumerator.DisposeAsync().AsTask() + : flow.CancelAsync(); + try + { + await blocker.WaitUntilContendedAsync(protocol.FlowControl.BackendProcessId); + while (flow.CancellationWindow < 1) + await Task.Yield(); + + var state = ProtocolDiag.CancellationState(protocol); + Assert.IsTrue(state.StartsWith("dispatching=True", StringComparison.Ordinal), + $"The successor cancellation did not dispatch at its read frontier: {state}"); + } + finally + { + settleAttempt.TrySetResult(CancelRequestState.Sent); + await blocker.ReleaseAsync(); + } + await cancellation; + if (!dispose) + { + await Assert.ThrowsExactlyAsync( + async () => await enumerator.MoveNextAsync()); + await enumerator.DisposeAsync(); + } + await WaitUntilAsync(() => !protocol.HasPendingCancellation); + await PgTestPool.RunAsync(protocol, "select 1"); + } + [TestMethod] [Ignore("Requires a second body-owned drain read after the consumer read times out; the replacement retains one read owner through drain.")] public async Task ServerCancel_ReadTimeoutAfterAmbiguousRetryAbortsWire() diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 385f7f5..2646634 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -1022,7 +1022,8 @@ bool TryTakeOverDrain() if (Interlocked.CompareExchange(ref _state.Phase, PhaseDraining, phase) != phase) continue; NotifyDrainStarted(); - _state.ConsumerDetached = true; + // Decoder takeover does not imply consumer abandonment. Explicit cancellation and + // graceful close also drain autonomously while retaining their consumer semantics. ThreadPool.UnsafeQueueUserWorkItem(static state => _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), _ops.Flow); @@ -1153,12 +1154,12 @@ internal ValueTask DisposeAsync() _state.ConsumerDetached = true; if (_state.Current is { IsComplete: false } || _state.CommandIndex + 1 < _state.Commands.Count) - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + RequestConsumerDrainCancellation(); return _state.WaitForDrainOnDispose ? DisposeDrainAsync() : FireAndForgetDrain(); case PhaseReading: _state.ConsumerDetached = true; NotifyDrainStarted(); - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + RequestConsumerDrainCancellation(); return _state.WaitForDrainOnDispose ? DisposeCompletedAsync() : default; default: return !_state.WaitForDrainOnDispose || _state.ConsumerObservedCompletion @@ -1217,7 +1218,7 @@ internal void Dispose() _state.ConsumerDetached = true; if (_state.Current is { IsComplete: false } || _state.CommandIndex + 1 < _state.Commands.Count) - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + RequestConsumerDrainCancellation(); Drain(); if (_state.WaitForDrainOnDispose) DisposeCompleted(); @@ -1225,7 +1226,7 @@ internal void Dispose() case PhaseReading: _state.ConsumerDetached = true; NotifyDrainStarted(); - RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow); + RequestConsumerDrainCancellation(); if (_state.WaitForDrainOnDispose) DisposeCompleted(); return; @@ -1270,8 +1271,13 @@ void ThrowDrainErrors() throw new AggregateException(errors); } - // When true, disposal waits for the drain to reach RFQ and for framework release. Otherwise it - // returns while the drain continues autonomously. + // Give the current window a chance to finish naturally. Once disposal advances into an unread + // successor, dispatch at its read frontier instead of paying the grace period for every window. + void RequestConsumerDrainCancellation() + => RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, + BackendCancellationTiming.AfterGrace, + BackendCancellationTiming.AtReadFrontier); + void RegisterCancellation(CancellationToken callerToken) { // Keep the second token/registration pair off ordinary flow objects. Default-token traffic From 1dfa1c23150ccc5e38250483a8df13da3fa1cb04 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 12:34:36 +0200 Subject: [PATCH 095/136] Close command flows activated across shutdown --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 2646634..8e4500c 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -373,6 +373,15 @@ void OnActivationSettled(bool onExecutorStrand) CompletePipelineTask(null, runContinuationsAsynchronously: true); return; } + // Termination propagation enumerates pipeline positions best-effort. A flow can cross from + // the in-flight store into the activated slot while that pass is being taken and miss it. + // Recheck after publishing readiness: OnStopping arbitrates PhaseInitial against a consumer's + // PhaseReading claim, so exactly one side owns the decoder and eventual pipeline completion. + if (_state.Context.StoppingToken.IsCancellationRequested) + { + OnStopping(_state.Context.FlowTerminationException); + return; + } // A cancel latched before activation may have released its caller already. The response // still has to reach RFQ, so drain it unless a consumer already owns the decoder. if (IsCancelRequested) From 33a660090467c5c4f84d66b6fce0ff0a7b0e8f3e Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 13:26:56 +0200 Subject: [PATCH 096/136] Keep command consumers off the executor strand --- Slon.Tests/Pg/SyncFlowHandoffTests.cs | 65 +++++++++++++++++++++++++++ Slon/Pg/Protocol/Flows/CommandFlow.cs | 13 +++--- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/Slon.Tests/Pg/SyncFlowHandoffTests.cs b/Slon.Tests/Pg/SyncFlowHandoffTests.cs index 89f9993..a731768 100644 --- a/Slon.Tests/Pg/SyncFlowHandoffTests.cs +++ b/Slon.Tests/Pg/SyncFlowHandoffTests.cs @@ -1,3 +1,4 @@ +using Draghi.Pipelining; using Slon.Pg; using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; @@ -34,6 +35,27 @@ sealed class WakeHolder internal FlowCallerInteractionCore Core; } + sealed class TrackingScheduler : PipelineScheduler + { + [ThreadStatic] + static bool _isExecuting; + + internal static bool IsExecuting => _isExecuting; + + public override void SubmitDetached( + Action action, object? state, bool preferLocal = true) + => PipelineScheduler.ThreadPool.SubmitDetached(static state => + { + var work = (Work)state!; + var prior = _isExecuting; + _isExecuting = true; + try { work.Action(work.State); } + finally { _isExecuting = prior; } + }, new Work(action, state), preferLocal); + + sealed record Work(Action Action, object? State); + } + static ref FlowCallerInteractionCore GetWakeCore(WakeHolder holder) => ref holder.Core; [ConnectionCreatingTestMethod] @@ -70,6 +92,49 @@ public async Task ConcurrentSyncAndAsync_NoSharedPromiseCollision() Assert.Fail($"concurrent sync/async raised {failure}"); } + [ConnectionCreatingTestMethod] + public async Task DeferredActivation_DoesNotResumeConsumerOnExecutorStrand() + { + var scheduler = new TrackingScheduler(); + await using var blocker = await PgAdvisoryLock.AcquireAsync(); + await using var protocol = await PgTestPool.NewIsolatedAsync(o => + { + o.ExecutionScheduler = scheduler; + o.ActivationScheduler = scheduler; + }); + + var first = protocol.Queue(new CommandFlow(async: true, blocker.WaitCommand)); + var second = protocol.Queue(new CommandFlow(async: true, Command.Create("select 1"))); + var firstDrain = Drain(first); + var secondDrain = DrainAndObserveScheduler(second); + + await blocker.WaitUntilContendedAsync(protocol.FlowControl.BackendProcessId); + await blocker.ReleaseAsync(); + + await firstDrain; + Assert.IsFalse(await secondDrain, + "Deferred activation resumed consumer code on the pipeline executor strand."); + + static async Task Drain(CommandFlow flow) + { + var enumerator = flow.GetAsyncEnumerator(); + while (await enumerator.MoveNextAsync()) + await enumerator.Current.DisposeAsync(); + await enumerator.DisposeAsync(); + } + + static async Task DrainAndObserveScheduler(CommandFlow flow) + { + var enumerator = flow.GetAsyncEnumerator(); + Assert.IsTrue(await enumerator.MoveNextAsync()); + var resumedOnExecutor = TrackingScheduler.IsExecuting; + await enumerator.Current.DisposeAsync(); + Assert.IsFalse(await enumerator.MoveNextAsync()); + await enumerator.DisposeAsync(); + return resumedOnExecutor; + } + } + [ConnectionCreatingTestMethod] public async Task PairedAsyncAndSync_NoSharedPromiseCollision() { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 8e4500c..0e6bf19 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -328,11 +328,11 @@ internal ValueTask ExecuteAuto(PgClientFlow.Context context) // consumer ever arrives. var activation = context.GetDecoderAsync().ConfigureAwait(false); if (activation.IsCompleted) - OnActivationSettled(onExecutorStrand: true); + OnActivationSettled(); else activation.UnsafeOnCompleted(static state => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)) - .OnActivationSettled(onExecutorStrand: false), _ops.Flow); + .OnActivationSettled(), _ops.Flow); return new(new FlowTasks(writeTask, new ValueTask((IValueTaskSource)_ops.Flow, _state.PipelineTaskSource.Version))); } @@ -346,10 +346,9 @@ ValueTask WriteCommandsResumable(PgClientFlow.Context context, bool appendSync) return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); } - // Runs on the executor strand when activation already settled, else on the activation dispatch. - // The executor strand never runs consumer code. An activation dispatch is a detached work item - // whose only remaining work is this wake, so the consumer may continue on it directly. - void OnActivationSettled(bool onExecutorStrand) + // Activation completion can run inline from pipeline advancement even when it was pending during + // ExecuteAuto. Never let ready publication resume consumer code on that internal executor strand. + void OnActivationSettled() { try { @@ -366,7 +365,7 @@ void OnActivationSettled(bool onExecutorStrand) if (IsCancelRequested) RequestBackendCancellation(); - if (!CompleteReady(null, runContinuationsAsynchronously: onExecutorStrand)) + if (!CompleteReady(null, runContinuationsAsynchronously: true)) { // Teardown released the consumer while this flow waited for its turn. Nothing reads the // response, the closing wire owns it. From 875915bea91e88db2240119a1bc1b6d88387941b Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 14:00:42 +0200 Subject: [PATCH 097/136] Schedule command readiness only off executor activation --- Slon.Tests/Pg/SyncFlowHandoffTests.cs | 81 +++++++++------------------ Slon/Pg/Protocol/Flows/CommandFlow.cs | 15 +++-- Slon/Pg/Protocol/PgClientFlow.cs | 8 ++- Slon/Pg/Protocol/PgClientProtocol.cs | 6 +- 4 files changed, 47 insertions(+), 63 deletions(-) diff --git a/Slon.Tests/Pg/SyncFlowHandoffTests.cs b/Slon.Tests/Pg/SyncFlowHandoffTests.cs index a731768..fe06219 100644 --- a/Slon.Tests/Pg/SyncFlowHandoffTests.cs +++ b/Slon.Tests/Pg/SyncFlowHandoffTests.cs @@ -38,22 +38,22 @@ sealed class WakeHolder sealed class TrackingScheduler : PipelineScheduler { [ThreadStatic] - static bool _isExecuting; + static TrackingScheduler? _executing; - internal static bool IsExecuting => _isExecuting; + internal bool IsExecuting => ReferenceEquals(_executing, this); public override void SubmitDetached( Action action, object? state, bool preferLocal = true) => PipelineScheduler.ThreadPool.SubmitDetached(static state => { var work = (Work)state!; - var prior = _isExecuting; - _isExecuting = true; + var prior = _executing; + _executing = work.Scheduler; try { work.Action(work.State); } - finally { _isExecuting = prior; } - }, new Work(action, state), preferLocal); + finally { _executing = prior; } + }, new Work(this, action, state), preferLocal); - sealed record Work(Action Action, object? State); + sealed record Work(TrackingScheduler Scheduler, Action Action, object? State); } static ref FlowCallerInteractionCore GetWakeCore(WakeHolder holder) => ref holder.Core; @@ -61,7 +61,13 @@ sealed record Work(Action Action, object? State); [ConnectionCreatingTestMethod] public async Task ConcurrentSyncAndAsync_NoSharedPromiseCollision() { - await using var protocol = await PgTestPool.NewIsolatedAsync(); + var executionScheduler = new TrackingScheduler(); + var activationScheduler = new TrackingScheduler(); + await using var protocol = await PgTestPool.NewIsolatedAsync(o => + { + o.ExecutionScheduler = executionScheduler; + o.ActivationScheduler = activationScheduler; + }); Exception? failure = null; void Capture(Exception ex) => Interlocked.CompareExchange(ref failure, ex, null); @@ -70,7 +76,19 @@ public async Task ConcurrentSyncAndAsync_NoSharedPromiseCollision() try { for (var i = 0; i < StressIterations && Volatile.Read(ref failure) is null; i++) - await PgTestPool.RunAsync(protocol, "select 1"); + { + var flow = protocol.Queue(new CommandFlow(async: true, Command.Create("select 1"))); + var enumerator = flow.GetAsyncEnumerator(); + var hasResult = await enumerator.MoveNextAsync(); + if (hasResult && executionScheduler.IsExecuting) + Capture(new InvalidOperationException( + $"Async consumer resumed on the pipeline executor strand; " + + $"activationWasDispatched={flow.ActivationWasDispatched}.\n" + + Environment.StackTrace)); + while (hasResult) + hasResult = await enumerator.MoveNextAsync(); + await enumerator.DisposeAsync(); + } } catch (Exception ex) { Capture(ex); } }); @@ -87,54 +105,11 @@ public async Task ConcurrentSyncAndAsync_NoSharedPromiseCollision() syncThread.Start(); await asyncLoop; - syncThread.Join(); + await Task.Run(syncThread.Join); if (failure is not null) Assert.Fail($"concurrent sync/async raised {failure}"); } - [ConnectionCreatingTestMethod] - public async Task DeferredActivation_DoesNotResumeConsumerOnExecutorStrand() - { - var scheduler = new TrackingScheduler(); - await using var blocker = await PgAdvisoryLock.AcquireAsync(); - await using var protocol = await PgTestPool.NewIsolatedAsync(o => - { - o.ExecutionScheduler = scheduler; - o.ActivationScheduler = scheduler; - }); - - var first = protocol.Queue(new CommandFlow(async: true, blocker.WaitCommand)); - var second = protocol.Queue(new CommandFlow(async: true, Command.Create("select 1"))); - var firstDrain = Drain(first); - var secondDrain = DrainAndObserveScheduler(second); - - await blocker.WaitUntilContendedAsync(protocol.FlowControl.BackendProcessId); - await blocker.ReleaseAsync(); - - await firstDrain; - Assert.IsFalse(await secondDrain, - "Deferred activation resumed consumer code on the pipeline executor strand."); - - static async Task Drain(CommandFlow flow) - { - var enumerator = flow.GetAsyncEnumerator(); - while (await enumerator.MoveNextAsync()) - await enumerator.Current.DisposeAsync(); - await enumerator.DisposeAsync(); - } - - static async Task DrainAndObserveScheduler(CommandFlow flow) - { - var enumerator = flow.GetAsyncEnumerator(); - Assert.IsTrue(await enumerator.MoveNextAsync()); - var resumedOnExecutor = TrackingScheduler.IsExecuting; - await enumerator.Current.DisposeAsync(); - Assert.IsFalse(await enumerator.MoveNextAsync()); - await enumerator.DisposeAsync(); - return resumedOnExecutor; - } - } - [ConnectionCreatingTestMethod] public async Task PairedAsyncAndSync_NoSharedPromiseCollision() { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 0e6bf19..8b5afd2 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -328,11 +328,12 @@ internal ValueTask ExecuteAuto(PgClientFlow.Context context) // consumer ever arrives. var activation = context.GetDecoderAsync().ConfigureAwait(false); if (activation.IsCompleted) - OnActivationSettled(); + OnActivationSettled(onExecutorStrand: true); else activation.UnsafeOnCompleted(static state => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)) - .OnActivationSettled(), _ops.Flow); + .OnActivationSettled(onExecutorStrand: + !((PgClientFlow)state!).ActivationWasDispatched), _ops.Flow); return new(new FlowTasks(writeTask, new ValueTask((IValueTaskSource)_ops.Flow, _state.PipelineTaskSource.Version))); } @@ -346,9 +347,10 @@ ValueTask WriteCommandsResumable(PgClientFlow.Context context, bool appendSync) return writeTask.IsCompleted ? writeTask : encoder.RunResumableTask(writeTask); } - // Activation completion can run inline from pipeline advancement even when it was pending during - // ExecuteAuto. Never let ready publication resume consumer code on that internal executor strand. - void OnActivationSettled() + // A detached activation callback may publish ready inline on that scheduler turn. ExecuteAuto + // can also observe an already-completed activation directly on the executor, while a pending + // zero-edge activation completes inline there; both executor cases need the scheduling firewall. + void OnActivationSettled(bool onExecutorStrand) { try { @@ -365,7 +367,8 @@ void OnActivationSettled() if (IsCancelRequested) RequestBackendCancellation(); - if (!CompleteReady(null, runContinuationsAsynchronously: true)) + var dispatchReady = _ops.IsAsyncAtDispatch && onExecutorStrand; + if (!CompleteReady(null, runContinuationsAsynchronously: dispatchReady)) { // Teardown released the consumer while this flow waited for its turn. Nothing reads the // response, the closing wire owns it. diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 5a163b8..369cdbd 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -132,7 +132,10 @@ internal virtual void Bind(PgClientFlowBindingContext? context) { } /// Initialize overwrote the first's item before its Execute ran (one pending activation /// per flow tenure makes the field safe). internal void PrepareActivationDispatch(PgClientProtocol.Control control) - => _pendingActivationControl = control; + { + _activationWasDispatched = true; + _pendingActivationControl = control; + } void IThreadPoolWorkItem.Execute() { @@ -160,6 +163,8 @@ private protected virtual void ExecuteDetachedWorkItem() int _cancellationWindow; internal int CancellationWindow => Volatile.Read(ref _cancellationWindow); bool _lastMessageInducesRfq; + bool _activationWasDispatched; + internal bool ActivationWasDispatched => _activationWasDispatched; // We store the IsAsync value at bind time so the protocol can keep track of pipeline stalls correctly. bool _isAsyncAtDispatch; // Tri-state int (0 = unset, 1 = true, 2 = false) instead of bool? so reads / writes can be @@ -449,6 +454,7 @@ public void Reset() _rfqCount = 0; _cancellationWindow = 0; _lastMessageInducesRfq = false; + _activationWasDispatched = false; HandoffEvent?.ResetPlacement(); _pendingTimeoutStarted = false; _enqueueOptions = FlowEnqueueOptions.None; diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index b29603d..cbcfdf5 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1398,9 +1398,9 @@ public void ActivateHeadItem(PgClientFlow item, bool preferAsync = true) // Only the body wake is deferred below. _control.BindDecoder(item); - // Inline-activate when the framework allows it (preferAsync=false) or the flow is sync: - // sync flows park on a kernel wait-handle signal, bounded cost, safe under the advancer - // latch. Async flows can attach arbitrary await continuations, so they go through TP. + // Inline-activate when the framework allows it (preferAsync=false) or the flow is sync. + // CommandFlow retains whether an async activation was dispatched so its consumer-facing + // ready publication supplies the scheduling firewall only when activation ran inline. if (preferAsync && item.IsAsyncAtDispatch) { // The flow itself is the work item: an immutable (flow, control) pairing per queued From 6bd8abc80cae035797d7ab38d127892327af13f7 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 14:44:47 +0200 Subject: [PATCH 098/136] Add reusable flow authoring contracts --- .../AutonomousFlowAuthoringTests.cs | 135 +++++++++++ .../ConsumerDrivenFlowAuthoringTests.cs | 209 ++++++++++++++++++ Slon.Tests/Pg/FlowAuthoring/README.md | 56 +++++ Slon.Tests/Pg/SyncFlowHandoffTests.cs | 74 ------- 4 files changed, 400 insertions(+), 74 deletions(-) create mode 100644 Slon.Tests/Pg/FlowAuthoring/AutonomousFlowAuthoringTests.cs create mode 100644 Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs create mode 100644 Slon.Tests/Pg/FlowAuthoring/README.md diff --git a/Slon.Tests/Pg/FlowAuthoring/AutonomousFlowAuthoringTests.cs b/Slon.Tests/Pg/FlowAuthoring/AutonomousFlowAuthoringTests.cs new file mode 100644 index 0000000..3ff8d78 --- /dev/null +++ b/Slon.Tests/Pg/FlowAuthoring/AutonomousFlowAuthoringTests.cs @@ -0,0 +1,135 @@ +using Slon.Pg; +using Slon.Pg.Protocol; +using Slon.Pg.Protocol.Flows; + +namespace Slon.Tests.Pg.FlowAuthoring; + +public interface IAutonomousFlowContract +{ + string Name { get; } + PgClientFlow Create(bool async, params Command[] commands); + bool WasReleased(PgClientFlow flow); +} + +sealed class AutonomousCommandFlowContract : IAutonomousFlowContract +{ + public static AutonomousCommandFlowContract Instance { get; } = new(); + public string Name => nameof(AutonomousCommandFlow); + + public PgClientFlow Create(bool async, params Command[] commands) + => new AutonomousCommandFlow(async, commands); + + public bool WasReleased(PgClientFlow flow) + => ((AutonomousCommandFlow)flow).Released; + + public override string ToString() => Name; +} + +sealed class AutonomousCommandFlow : PgClientFlow +{ + readonly CommandList _commands; + + internal AutonomousCommandFlow(bool async, params Command[] commands) + : base(supportsDeferredFlush: true) + { + IsAsync = async; + _commands = new(commands); + } + + internal bool Released { get; private set; } + + protected override ValueTask ExecuteAuto(Context context) + { + var write = _commands.WriteCommandsAsync(context.GetEncoder(), appendSync: true); + return new(new FlowTasks(write, DrainAsync(context))); + + static async ValueTask DrainAsync(Context context) + { + var decoder = await context.GetDecoderAsync().ConfigureAwait(false); + while (context.OutstandingRfqCount is not 0) + _ = await decoder.GetNextAsync().ConfigureAwait(false); + } + } + + protected override void OnReleasing(Exception? exception) => Released = true; +} + +[TestClass] +public class AutonomousFlowAuthoringTests : ConnectionCreatingTest +{ + public static IEnumerable Implementations + { + get + { + yield return [AutonomousCommandFlowContract.Instance]; + } + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task NaturalCompletion_ReleasesAfterPipelineTask( + IAutonomousFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: true, + Command.Create("select 1"), Command.Create("select 2"))); + + await flow.WaitForComplete(); + + Assert.IsTrue(contract.WasReleased(flow)); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task SynchronousMode_DoesNotRequireCallerHandoff( + IAutonomousFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: false, Command.Create("select 1"))); + + await flow.WaitForComplete(); + + Assert.IsFalse(flow.NeedsSyncHandoff); + await PgTestPool.RunAsync(protocol, "select 2"); + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task ForcefulAbort_CompletesWithoutExternalDriver( + IAutonomousFlowContract contract) + { + await using var blocker = await PgAdvisoryLock.AcquireAsync(); + var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: true, blocker.WaitCommand)); + try + { + await blocker.WaitUntilContendedAsync(protocol.FlowControl.BackendProcessId); + await protocol.CompleteAsync(new IOException("contract abort")); + await Assert.ThrowsExactlyAsync( + async () => await flow.WaitForComplete()); + } + finally + { + await blocker.ReleaseAsync(); + await protocol.DisposeAsync(); + } + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task PipelinedInstances_CompleteAndRelease( + IAutonomousFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var first = protocol.Queue(contract.Create(async: true, Command.Create("select 1"))); + var second = protocol.Queue(contract.Create(async: true, Command.Create("select 2"))); + + await first.WaitForComplete(); + await second.WaitForComplete(); + + Assert.IsTrue(contract.WasReleased(first)); + Assert.IsTrue(contract.WasReleased(second)); + await PgTestPool.RunAsync(protocol, "select 3"); + } +} diff --git a/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs b/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs new file mode 100644 index 0000000..6add517 --- /dev/null +++ b/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs @@ -0,0 +1,209 @@ +using Draghi.Pipelining; +using Slon.Pg; +using Slon.Pg.Protocol; +using Slon.Pg.Protocol.Flows; + +namespace Slon.Tests.Pg.FlowAuthoring; + +public interface IConsumerDrivenFlowContract +{ + string Name { get; } + PgClientFlow Create(bool async, params Command[] commands); + IEnumerator GetEnumerator(PgClientFlow flow); + IAsyncEnumerator GetAsyncEnumerator(PgClientFlow flow); +} + +sealed class CommandFlowContract : IConsumerDrivenFlowContract +{ + public static CommandFlowContract Instance { get; } = new(); + public string Name => nameof(CommandFlow); + + public PgClientFlow Create(bool async, params Command[] commands) + => new CommandFlow(async, commands); + + public IEnumerator GetEnumerator(PgClientFlow flow) + => ((CommandFlow)flow).GetEnumerator(); + + public IAsyncEnumerator GetAsyncEnumerator(PgClientFlow flow) + => ((CommandFlow)flow).GetAsyncEnumerator(); + + public override string ToString() => Name; +} + +[TestClass] +public class ConsumerDrivenFlowAuthoringTests : ConnectionCreatingTest +{ + public static IEnumerable Implementations + { + get + { + yield return [CommandFlowContract.Instance]; + } + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task AsyncNaturalExhaustion_ReleasesWire(IConsumerDrivenFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: true, + Command.Create("select 1"), Command.Create("select 2"))); + + var results = contract.GetAsyncEnumerator(flow); + while (await results.MoveNextAsync()) + await results.Current.DisposeAsync(); + await results.DisposeAsync(); + + await flow.WaitForComplete(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task SyncNaturalExhaustion_ReleasesWire(IConsumerDrivenFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: false, + Command.Create("select 1"), Command.Create("select 2"))); + + using (var results = contract.GetEnumerator(flow)) + { + while (results.MoveNext()) + results.Current.Dispose(); + } + + await flow.WaitForComplete(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task AsyncAbandonBeforeRead_DrainsAndReleasesWire(IConsumerDrivenFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: true, + Command.Create("select generate_series(1, 20)"), Command.Create("select 2"))); + + await contract.GetAsyncEnumerator(flow).DisposeAsync(); + + await flow.WaitForComplete(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task AsyncAbandonAfterPublication_DrainsAndReleasesWire(IConsumerDrivenFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: true, + Command.Create("select generate_series(1, 20)"), Command.Create("select 2"))); + var results = contract.GetAsyncEnumerator(flow); + + Assert.IsTrue(await results.MoveNextAsync()); + await results.DisposeAsync(); + + await flow.WaitForComplete(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task AbortWithoutConsumer_CompletesFlow(IConsumerDrivenFlowContract contract) + { + await using var blocker = await PgAdvisoryLock.AcquireAsync(); + var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = protocol.Queue(contract.Create(async: true, blocker.WaitCommand)); + try + { + await blocker.WaitUntilContendedAsync(protocol.FlowControl.BackendProcessId); + await protocol.CompleteAsync(new IOException("contract abort")); + await Assert.ThrowsExactlyAsync( + async () => await flow.WaitForComplete()); + } + finally + { + await blocker.ReleaseAsync(); + await protocol.DisposeAsync(); + } + } + + [TestMethod] + [DynamicData(nameof(Implementations))] + public async Task MixedSyncAndAsync_NeverRunsConsumerOnExecutor( + IConsumerDrivenFlowContract contract) + { + var executionScheduler = new TrackingScheduler(); + var activationScheduler = new TrackingScheduler(); + await using var protocol = await PgTestPool.NewIsolatedAsync(o => + { + o.ExecutionScheduler = executionScheduler; + o.ActivationScheduler = activationScheduler; + }); + Exception? failure = null; + void Capture(Exception exception) + => Interlocked.CompareExchange(ref failure, exception, null); + + var asyncLoop = Task.Run(async () => + { + try + { + for (var i = 0; i < StressEnv.Iterations(32, 8_000) + && Volatile.Read(ref failure) is null; i++) + { + var flow = protocol.Queue(contract.Create(async: true, Command.Create("select 1"))); + var results = contract.GetAsyncEnumerator(flow); + var hasResult = await results.MoveNextAsync(); + if (hasResult && executionScheduler.IsExecuting) + Capture(new InvalidOperationException( + $"{contract.Name} resumed its async consumer on the pipeline executor.")); + while (hasResult) + hasResult = await results.MoveNextAsync(); + await results.DisposeAsync(); + } + } + catch (Exception ex) { Capture(ex); } + }); + + var syncThread = new Thread(() => + { + try + { + for (var i = 0; i < StressEnv.Iterations(32, 8_000) + && Volatile.Read(ref failure) is null; i++) + { + var flow = protocol.Queue(contract.Create(async: false, Command.Create("select 1"))); + using var results = contract.GetEnumerator(flow); + while (results.MoveNext()) { } + } + } + catch (Exception ex) { Capture(ex); } + }) { IsBackground = true, Name = $"{contract.Name}-sync-contract" }; + + syncThread.Start(); + await asyncLoop; + await Task.Run(syncThread.Join); + if (failure is not null) + Assert.Fail($"{contract.Name} violated mixed-consumer execution: {failure}"); + } + + sealed class TrackingScheduler : PipelineScheduler + { + [ThreadStatic] + static TrackingScheduler? _executing; + + internal bool IsExecuting => ReferenceEquals(_executing, this); + + public override void SubmitDetached( + Action action, object? state, bool preferLocal = true) + => PipelineScheduler.ThreadPool.SubmitDetached(static state => + { + var work = (Work)state!; + var prior = _executing; + _executing = work.Scheduler; + try { work.Action(work.State); } + finally { _executing = prior; } + }, new Work(this, action, state), preferLocal); + + sealed record Work(TrackingScheduler Scheduler, Action Action, object? State); + } +} diff --git a/Slon.Tests/Pg/FlowAuthoring/README.md b/Slon.Tests/Pg/FlowAuthoring/README.md new file mode 100644 index 0000000..209f345 --- /dev/null +++ b/Slon.Tests/Pg/FlowAuthoring/README.md @@ -0,0 +1,56 @@ +# PostgreSQL flow authoring contracts + +These tests are the executable lifecycle specification for `PgClientFlow` +implementations. They exercise behavior shared by flow types; SQL semantics and +result-shape tests remain with the implementation. + +## Profiles + +- `IAutonomousFlowContract` applies when the flow owns its complete pipeline + task and needs no external consumer to reach RFQ. +- `IConsumerDrivenFlowContract` applies when an external sync or async consumer + owns result advancement and can abandon that ownership. + +A flow exposing both kinds of entry point should register an adapter for every +profile it supports. Profiles are separate interfaces so an implementation +cannot silently opt out of individual required scenarios with capability flags. + +## Registering a flow + +Implement the relevant adapter in this directory and add its singleton to the +profile's `Implementations` data source. For a consumer-driven flow the complete +adapter is: + +```csharp +sealed class MyFlowContract : IConsumerDrivenFlowContract +{ + public string Name => nameof(MyFlow); + public PgClientFlow Create(bool async, params Command[] commands) => + new MyFlow(async, commands); + public IEnumerator GetEnumerator(PgClientFlow flow) => + ((MyFlow)flow).GetEnumerator(); + public IAsyncEnumerator GetAsyncEnumerator(PgClientFlow flow) => + ((MyFlow)flow).GetAsyncEnumerator(); +} +``` + +The interface conversions may box a value-type enumerator. This suite verifies +correctness rather than allocation behavior; implementation-specific benchmarks +continue to call concrete enumerators directly. + +## Contract rules + +The suite treats submission, activation, consumer progress, and protocol +lifecycle as independent axes. In particular: + +- the pipeline task retains wire/decoder tenure through RFQ; +- consumer abandonment transfers read ownership and leaves the wire reusable; +- forceful termination completes a flow even if no consumer attaches; +- sync execution blocks only its caller and does not require a handoff for an + autonomous flow; +- an async consumer never runs on the pipeline executor strand; +- release occurs before completion makes a flow reusable. + +Contract scenarios must use explicit scheduler gates, in-memory transports, or +`FakeTimeProvider` when ordering matters. Do not use wall-clock delays or +timeouts as correctness oracles. diff --git a/Slon.Tests/Pg/SyncFlowHandoffTests.cs b/Slon.Tests/Pg/SyncFlowHandoffTests.cs index fe06219..cbeb7db 100644 --- a/Slon.Tests/Pg/SyncFlowHandoffTests.cs +++ b/Slon.Tests/Pg/SyncFlowHandoffTests.cs @@ -1,4 +1,3 @@ -using Draghi.Pipelining; using Slon.Pg; using Slon.Pg.Protocol; using Slon.Pg.Protocol.Flows; @@ -35,81 +34,8 @@ sealed class WakeHolder internal FlowCallerInteractionCore Core; } - sealed class TrackingScheduler : PipelineScheduler - { - [ThreadStatic] - static TrackingScheduler? _executing; - - internal bool IsExecuting => ReferenceEquals(_executing, this); - - public override void SubmitDetached( - Action action, object? state, bool preferLocal = true) - => PipelineScheduler.ThreadPool.SubmitDetached(static state => - { - var work = (Work)state!; - var prior = _executing; - _executing = work.Scheduler; - try { work.Action(work.State); } - finally { _executing = prior; } - }, new Work(this, action, state), preferLocal); - - sealed record Work(TrackingScheduler Scheduler, Action Action, object? State); - } - static ref FlowCallerInteractionCore GetWakeCore(WakeHolder holder) => ref holder.Core; - [ConnectionCreatingTestMethod] - public async Task ConcurrentSyncAndAsync_NoSharedPromiseCollision() - { - var executionScheduler = new TrackingScheduler(); - var activationScheduler = new TrackingScheduler(); - await using var protocol = await PgTestPool.NewIsolatedAsync(o => - { - o.ExecutionScheduler = executionScheduler; - o.ActivationScheduler = activationScheduler; - }); - Exception? failure = null; - void Capture(Exception ex) => Interlocked.CompareExchange(ref failure, ex, null); - - var asyncLoop = Task.Run(async () => - { - try - { - for (var i = 0; i < StressIterations && Volatile.Read(ref failure) is null; i++) - { - var flow = protocol.Queue(new CommandFlow(async: true, Command.Create("select 1"))); - var enumerator = flow.GetAsyncEnumerator(); - var hasResult = await enumerator.MoveNextAsync(); - if (hasResult && executionScheduler.IsExecuting) - Capture(new InvalidOperationException( - $"Async consumer resumed on the pipeline executor strand; " + - $"activationWasDispatched={flow.ActivationWasDispatched}.\n" + - Environment.StackTrace)); - while (hasResult) - hasResult = await enumerator.MoveNextAsync(); - await enumerator.DisposeAsync(); - } - } - catch (Exception ex) { Capture(ex); } - }); - - var syncThread = new Thread(() => - { - try - { - for (var i = 0; i < StressIterations && Volatile.Read(ref failure) is null; i++) - PgTestPool.RunSync(protocol, "select 1").GetAwaiter().GetResult(); - } - catch (Exception ex) { Capture(ex); } - }) { IsBackground = true, Name = "sync-flow-handoff" }; - - syncThread.Start(); - await asyncLoop; - await Task.Run(syncThread.Join); - if (failure is not null) - Assert.Fail($"concurrent sync/async raised {failure}"); - } - [ConnectionCreatingTestMethod] public async Task PairedAsyncAndSync_NoSharedPromiseCollision() { From 460d7fca6d12431e53fca53a06a78f090ee60c2c Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:08:30 +0200 Subject: [PATCH 099/136] Cover command flow batching --- Slon.Tests/Pg/CommandFlowBatchTests.cs | 154 +++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 Slon.Tests/Pg/CommandFlowBatchTests.cs diff --git a/Slon.Tests/Pg/CommandFlowBatchTests.cs b/Slon.Tests/Pg/CommandFlowBatchTests.cs new file mode 100644 index 0000000..d37792a --- /dev/null +++ b/Slon.Tests/Pg/CommandFlowBatchTests.cs @@ -0,0 +1,154 @@ +using Slon.Pg; +using Slon.Pg.Protocol; +using Slon.Pg.Protocol.Flows; +using Slon.Text; + +namespace Slon.Tests.Pg; + +[TestClass] +public class CommandFlowBatchTests +{ + static async Task Prepare(PgClientProtocol protocol, string sql, EncodedCString name) + { + var flow = protocol.Queue(new CommandFlow(async: true, + Command.Create(sql, commandName: name) with { DescribeOnly = true })); + var results = flow.GetAsyncEnumerator(); + CommandDescriptor descriptor = default; + while (await results.MoveNextAsync()) + descriptor = results.Current.GetMetadata().ToPreparedDescriptor(); + await results.DisposeAsync(); + return descriptor; + } + + [ConnectionCreatingTestMethod] + public async Task Batch_TwoPreparedCommands_EnumeratesResultsInOrder() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var first = await Prepare(protocol, "select 1::int4", "batch_1"); + var second = await Prepare(protocol, "select 2::int4", "batch_2"); + var results = protocol.Queue(new CommandFlow(async: true, new CommandList( + Command.Create(first), Command.Create(second)))).GetAsyncEnumerator(); + var values = new List(); + while (await results.MoveNextAsync()) + { + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync()) + values.Add(rows.Current.GetValue(0)); + await rows.DisposeAsync(); + } + await results.DisposeAsync(); + CollectionAssert.AreEqual(new[] { 1, 2 }, values); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [ConnectionCreatingTestMethod] + public async Task Batch_DisposeBeforeRead_DrainsEveryCommand() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var first = await Prepare(protocol, "select generate_series(1, 20)", "batch_dispose_1"); + var second = await Prepare(protocol, "select generate_series(1, 20)", "batch_dispose_2"); + var results = protocol.Queue(new CommandFlow(async: true, new CommandList( + Command.Create(first), Command.Create(second)))).GetAsyncEnumerator(); + await results.DisposeAsync(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [ConnectionCreatingTestMethod] + public async Task Batch_ErrorWithoutBarrier_DiscardsSuccessorThroughFinalSync() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var results = protocol.Queue(new CommandFlow(async: true, new CommandList( + Command.Create("select 1 / 0"), Command.Create("select 2::int4")))) + .GetAsyncEnumerator(); + Assert.IsTrue(await results.MoveNextAsync()); + var rows = results.Current.GetAsyncEnumerator(); + Assert.IsFalse(await rows.MoveNextAsync()); + await rows.DisposeAsync(); + Assert.ThrowsExactly(() => results.Current.GetCommandComplete()); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [ConnectionCreatingTestMethod] + public async Task Batch_ErrorBarrier_AllowsFollowingCommand() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var results = protocol.Queue(new CommandFlow(async: true, new CommandList( + Command.Create("select 1 / 0") with { WithSync = true }, + Command.Create("select 2::int4")))) + .GetAsyncEnumerator(); + Assert.IsTrue(await results.MoveNextAsync()); + var failedRows = results.Current.GetAsyncEnumerator(); + Assert.IsFalse(await failedRows.MoveNextAsync()); + await failedRows.DisposeAsync(); + Assert.ThrowsExactly(() => results.Current.GetCommandComplete()); + Assert.IsTrue(await results.MoveNextAsync()); + var rows = results.Current.GetAsyncEnumerator(); + Assert.IsTrue(await rows.MoveNextAsync()); + Assert.AreEqual(2, rows.Current.GetValue(0)); + Assert.IsFalse(await rows.MoveNextAsync()); + await rows.DisposeAsync(); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [ConnectionCreatingTestMethod] + public async Task Execution_SyncTwoPreparedCommands_EnumeratesResultsInOrder() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var first = await Prepare(protocol, "select 1::int4", "execution_sync_1"); + var second = await Prepare(protocol, "select 2::int4", "execution_sync_2"); + var results = protocol.Queue(new CommandFlow(async: false, new CommandList( + Command.Create(first), Command.Create(second)))).GetEnumerator(); + var values = new List(); + while (results.MoveNext()) + { + using var rows = results.Current.GetEnumerator(); + while (rows.MoveNext()) + values.Add(rows.Current.GetValue(0)); + } + results.Dispose(); + CollectionAssert.AreEqual(new[] { 1, 2 }, values); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [ConnectionCreatingTestMethod] + public async Task Execution_SyncDisposeBeforeRead_DrainsEveryCommand() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var first = await Prepare(protocol, "select generate_series(1, 20)", "execution_sync_dispose_1"); + var second = await Prepare(protocol, "select generate_series(1, 20)", "execution_sync_dispose_2"); + var results = protocol.Queue(new CommandFlow(async: false, new CommandList( + Command.Create(first), Command.Create(second)))).GetEnumerator(); + results.Dispose(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + + [ConnectionCreatingTestMethod] + public async Task Execution_SyncErrorBarrier_AllowsFollowingCommand() + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var results = protocol.Queue(new CommandFlow(async: false, new CommandList( + Command.Create("select 1 / 0") with { WithSync = true }, + Command.Create("select 2::int4")))).GetEnumerator(); + Assert.IsTrue(results.MoveNext()); + using (var failedRows = results.Current.GetEnumerator()) + Assert.IsFalse(failedRows.MoveNext()); + Assert.ThrowsExactly(() => results.Current.GetCommandComplete()); + Assert.IsTrue(results.MoveNext()); + using (var rows = results.Current.GetEnumerator()) + { + Assert.IsTrue(rows.MoveNext()); + Assert.AreEqual(2, rows.Current.GetValue(0)); + Assert.IsFalse(rows.MoveNext()); + } + Assert.IsFalse(results.MoveNext()); + results.Dispose(); + await PgTestPool.RunAsync(protocol, "select 3"); + } + +} + + From 9134da26b83db0135a0797e555bcc62858dedbeb Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:14:01 +0200 Subject: [PATCH 100/136] Expand reusable flow authoring contracts --- .../Pg/CommandResultEnumerationTests.cs | 24 --- .../ConsumerDrivenFlowAuthoringTests.cs | 31 +++- ...onsumerDrivenFlowSemanticContractTests.cs} | 174 ++++++++++++------ Slon.Tests/Pg/FlowAuthoring/README.md | 5 +- 4 files changed, 152 insertions(+), 82 deletions(-) rename Slon.Tests/Pg/{CommandFlowContractTests.cs => FlowAuthoring/ConsumerDrivenFlowSemanticContractTests.cs} (64%) diff --git a/Slon.Tests/Pg/CommandResultEnumerationTests.cs b/Slon.Tests/Pg/CommandResultEnumerationTests.cs index 2aa8c31..1a8ab21 100644 --- a/Slon.Tests/Pg/CommandResultEnumerationTests.cs +++ b/Slon.Tests/Pg/CommandResultEnumerationTests.cs @@ -236,28 +236,4 @@ public async Task ErrorWithoutSync_ResumesAfterInternalSync(bool async) await PgTestPool.RunAsync(protocol, "select 1"); } - // Reset must clear the terminal enumeration state, or a reused flow reports exhaustion before - // its next tenure publishes anything. - [ConnectionCreatingTestMethod] - public async Task Reset_ClearsEnumerationCompleted_ForNextTenure() - { - await using var protocol = await PgTestPool.NewIsolatedAsync(); - var flow = new CommandFlow( - async: true, enableActivationTimeout: false, Command.Create("select 1")); - for (var tenure = 0; tenure < 2; tenure++) - { - if (tenure > 0) - { - flow.Reset(); - flow.Initialize(async: true, Command.Create("select 2")); - } - protocol.Queue(flow); - var e = flow.GetAsyncEnumerator(); - Assert.IsTrue(await e.MoveNextAsync(), $"tenure {tenure} must publish its result"); - await e.Current.DisposeAsync(); - Assert.IsFalse(await e.MoveNextAsync()); - await e.DisposeAsync(); - } - } - } diff --git a/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs b/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs index 6add517..a421c5c 100644 --- a/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs +++ b/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowAuthoringTests.cs @@ -11,9 +11,17 @@ public interface IConsumerDrivenFlowContract PgClientFlow Create(bool async, params Command[] commands); IEnumerator GetEnumerator(PgClientFlow flow); IAsyncEnumerator GetAsyncEnumerator(PgClientFlow flow); + ValueTask MoveNextAsync( + IAsyncEnumerator results, CancellationToken cancellationToken); } -sealed class CommandFlowContract : IConsumerDrivenFlowContract +public interface IReusableConsumerDrivenFlowContract : IConsumerDrivenFlowContract +{ + PgClientFlow CreateReusable(bool async, params Command[] commands); + void Reset(PgClientFlow flow, bool async, params Command[] commands); +} + +sealed class CommandFlowContract : IReusableConsumerDrivenFlowContract { public static CommandFlowContract Instance { get; } = new(); public string Name => nameof(CommandFlow); @@ -27,6 +35,19 @@ public IEnumerator GetEnumerator(PgClientFlow flow) public IAsyncEnumerator GetAsyncEnumerator(PgClientFlow flow) => ((CommandFlow)flow).GetAsyncEnumerator(); + public ValueTask MoveNextAsync( + IAsyncEnumerator results, CancellationToken cancellationToken) + => ((CommandFlow.Enumerator)results).MoveNextAsync(cancellationToken); + + public PgClientFlow CreateReusable(bool async, params Command[] commands) + => new CommandFlow(async, enableActivationTimeout: false, commands); + + public void Reset(PgClientFlow flow, bool async, params Command[] commands) + { + flow.Reset(); + ((CommandFlow)flow).Initialize(async, commands); + } + public override string ToString() => Name; } @@ -41,6 +62,14 @@ public static IEnumerable Implementations } } + public static IEnumerable ReusableImplementations + { + get + { + yield return [CommandFlowContract.Instance]; + } + } + [TestMethod] [DynamicData(nameof(Implementations))] public async Task AsyncNaturalExhaustion_ReleasesWire(IConsumerDrivenFlowContract contract) diff --git a/Slon.Tests/Pg/CommandFlowContractTests.cs b/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowSemanticContractTests.cs similarity index 64% rename from Slon.Tests/Pg/CommandFlowContractTests.cs rename to Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowSemanticContractTests.cs index 54488a6..fe8f054 100644 --- a/Slon.Tests/Pg/CommandFlowContractTests.cs +++ b/Slon.Tests/Pg/FlowAuthoring/ConsumerDrivenFlowSemanticContractTests.cs @@ -5,17 +5,35 @@ using Slon.Pg.Types; using Slon.Text; -namespace Slon.Tests.Pg; +namespace Slon.Tests.Pg.FlowAuthoring; // Command-result semantics shared by consumer-driven PostgreSQL flows. [TestClass] -public class CommandFlowContractTests +public class ConsumerDrivenFlowSemanticContractTests { + public static IEnumerable Implementations + => ConsumerDrivenFlowAuthoringTests.Implementations; + + public static IEnumerable ReusableImplementations + => ConsumerDrivenFlowAuthoringTests.ReusableImplementations; + + public static IEnumerable PreparedCases + { + get + { + foreach (var implementation in Implementations) + { + yield return [implementation[0], 0]; + yield return [implementation[0], 3]; + } + } + } static async Task Prepare( + IConsumerDrivenFlowContract contract, PgClientProtocol protocol, string sql, EncodedCString name) { - var results = protocol.Queue(new CommandFlow(async: true, - Command.Create(sql, commandName: name) with { DescribeOnly = true })).GetAsyncEnumerator(); + var results = Queue(contract, protocol, + Command.Create(sql, commandName: name) with { DescribeOnly = true }); CommandDescriptor descriptor = default; while (await results.MoveNextAsync()) descriptor = results.Current.GetMetadata().ToPreparedDescriptor(); @@ -23,10 +41,13 @@ static async Task Prepare( return descriptor; } - static Results Queue(PgClientProtocol protocol, in Command command, + static Results Queue(IConsumerDrivenFlowContract contract, + PgClientProtocol protocol, in Command command, CancellationToken cancellationToken = default) - => new(protocol.Queue(new CommandFlow(async: true, command), cancellationToken) - .GetAsyncEnumerator()); + { + var flow = protocol.Queue(contract.Create(async: true, command), cancellationToken); + return new(contract, contract.GetAsyncEnumerator(flow)); + } static async Task CountRows(Results results) { @@ -43,36 +64,38 @@ static async Task CountRows(Results results) } [ConnectionCreatingTestMethod] - [DataRow(0)] - [DataRow(3)] - public async Task Prepared_NaturalExhaustion(int rowCount) + [DynamicData(nameof(PreparedCases))] + public async Task Prepared_NaturalExhaustion( + IConsumerDrivenFlowContract contract, int rowCount) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - var descriptor = await Prepare(protocol, + var descriptor = await Prepare(contract, protocol, $"select generate_series(1, {rowCount})", $"contract_rows_{rowCount}"); Assert.AreEqual(rowCount, - await CountRows(Queue(protocol, Command.Create(descriptor)))); + await CountRows(Queue(contract, protocol, Command.Create(descriptor)))); await PgTestPool.RunAsync(protocol, "select 1"); } [ConnectionCreatingTestMethod] - public async Task Unprepared_NaturalExhaustion() + [DynamicData(nameof(Implementations))] + public async Task Unprepared_NaturalExhaustion(IConsumerDrivenFlowContract contract) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - Assert.AreEqual(2, await CountRows(Queue(protocol, + Assert.AreEqual(2, await CountRows(Queue(contract, protocol, Command.Create("select generate_series(1, 2)")))); await PgTestPool.RunAsync(protocol, "select 1"); } [ConnectionCreatingTestMethod] - public async Task DisposeBeforeAnyRead_DrainsAndKeepsWire() + [DynamicData(nameof(Implementations))] + public async Task DisposeBeforeAnyRead_DrainsAndKeepsWire(IConsumerDrivenFlowContract contract) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - var descriptor = await Prepare(protocol, + var descriptor = await Prepare(contract, protocol, "select generate_series(1, 1000)", "contract_unread"); - var results = Queue(protocol, Command.Create(descriptor)); + var results = Queue(contract, protocol, Command.Create(descriptor)); await results.DisposeAsync(); @@ -80,12 +103,13 @@ public async Task DisposeBeforeAnyRead_DrainsAndKeepsWire() } [ConnectionCreatingTestMethod] - public async Task DisposeAfterOneRow_DrainsAndKeepsWire() + [DynamicData(nameof(Implementations))] + public async Task DisposeAfterOneRow_DrainsAndKeepsWire(IConsumerDrivenFlowContract contract) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - var descriptor = await Prepare(protocol, + var descriptor = await Prepare(contract, protocol, "select generate_series(1, 20000)", "contract_partial"); - var results = Queue(protocol, Command.Create(descriptor)); + var results = Queue(contract, protocol, Command.Create(descriptor)); Assert.IsTrue(await results.MoveNextAsync()); var rows = results.Current.GetAsyncEnumerator(); @@ -96,11 +120,12 @@ public async Task DisposeAfterOneRow_DrainsAndKeepsWire() } [ConnectionCreatingTestMethod] - public async Task CommandError_IsResultAndKeepsWire() + [DynamicData(nameof(Implementations))] + public async Task CommandError_IsResultAndKeepsWire(IConsumerDrivenFlowContract contract) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - var descriptor = await Prepare(protocol, "select 1 / 0", "contract_error"); - var results = Queue(protocol, Command.Create(descriptor)); + var descriptor = await Prepare(contract, protocol, "select 1 / 0", "contract_error"); + var results = Queue(contract, protocol, Command.Create(descriptor)); Assert.IsTrue(await results.MoveNextAsync()); var failed = results.Current; @@ -115,11 +140,12 @@ public async Task CommandError_IsResultAndKeepsWire() } [ConnectionCreatingTestMethod] - public async Task PreparedMetadataAndCompletion_Agree() + [DynamicData(nameof(Implementations))] + public async Task PreparedMetadataAndCompletion_Agree(IConsumerDrivenFlowContract contract) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - var descriptor = await Prepare(protocol, "select 42::int4", "contract_metadata"); - var results = Queue(protocol, Command.Create(descriptor)); + var descriptor = await Prepare(contract, protocol, "select 42::int4", "contract_metadata"); + var results = Queue(contract, protocol, Command.Create(descriptor)); Assert.IsTrue(await results.MoveNextAsync()); var result = results.Current; @@ -141,34 +167,38 @@ public async Task PreparedMetadataAndCompletion_Agree() } [ConnectionCreatingTestMethod] - public async Task CancellationWhileReadPending_DeliversTokenAndKeepsWire() + [DynamicData(nameof(Implementations))] + public async Task CancellationWhileReadPending_DeliversTokenAndKeepsWire(IConsumerDrivenFlowContract contract) { + await using var blocker = await PgAdvisoryLock.AcquireAsync(); await using var protocol = await NewCancelableProtocolAsync(); - var descriptor = await Prepare(protocol, "select pg_sleep(30)", "contract_cancel_pending"); using var cancellation = new CancellationTokenSource(); - var results = Queue(protocol, Command.Create(descriptor)); + var results = Queue(contract, protocol, blocker.WaitCommand); var pending = results.MoveNextAsync(cancellation.Token); Assert.IsFalse(pending.IsCompleted); - cancellation.CancelAfter(TimeSpan.FromMilliseconds(200)); + await blocker.WaitUntilContendedAsync(protocol.FlowControl.BackendProcessId); + cancellation.Cancel(); var exception = await Assert.ThrowsExactlyAsync( async () => await pending); Assert.AreEqual(cancellation.Token, exception.CancellationToken); await Assert.ThrowsExactlyAsync( async () => await results.MoveNextAsync()); await results.DisposeAsync(); + await blocker.ReleaseAsync(); await PgTestPool.RunAsync(protocol, "select 1"); } [ConnectionCreatingTestMethod] - public async Task CancellationAfterRow_DrainsAndKeepsWire() + [DynamicData(nameof(Implementations))] + public async Task CancellationAfterRow_DrainsAndKeepsWire(IConsumerDrivenFlowContract contract) { await using var protocol = await NewCancelableProtocolAsync(); - var descriptor = await Prepare(protocol, + var descriptor = await Prepare(contract, protocol, "select generate_series(1, 20000)", "contract_cancel_row"); using var cancellation = new CancellationTokenSource(); - var results = Queue(protocol, Command.Create(descriptor)); + var results = Queue(contract, protocol, Command.Create(descriptor)); Assert.IsTrue(await results.MoveNextAsync(cancellation.Token)); var rows = results.Current.GetAsyncEnumerator(); @@ -182,12 +212,13 @@ await Assert.ThrowsExactlyAsync( } [ConnectionCreatingTestMethod] - public async Task PreCancelledRead_ReleasesCallerAndKeepsWire() + [DynamicData(nameof(Implementations))] + public async Task PreCancelledRead_ReleasesCallerAndKeepsWire(IConsumerDrivenFlowContract contract) { await using var protocol = await NewCancelableProtocolAsync(); - var descriptor = await Prepare(protocol, + var descriptor = await Prepare(contract, protocol, "select generate_series(1, 1000)", "contract_precancel"); - var results = Queue(protocol, Command.Create(descriptor)); + var results = Queue(contract, protocol, Command.Create(descriptor)); var cancellationToken = new CancellationToken(canceled: true); var exception = await Assert.ThrowsExactlyAsync( @@ -201,13 +232,14 @@ await Assert.ThrowsExactlyAsync( } [ConnectionCreatingTestMethod] - public async Task SuccessorProgressesAfterAbandonment() + [DynamicData(nameof(Implementations))] + public async Task SuccessorProgressesAfterAbandonment(IConsumerDrivenFlowContract contract) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - var descriptor = await Prepare(protocol, + var descriptor = await Prepare(contract, protocol, "select generate_series(1, 20000)", "contract_successor"); - var first = Queue(protocol, Command.Create(descriptor)); - var second = Queue(protocol, Command.Create(descriptor)); + var first = Queue(contract, protocol, Command.Create(descriptor)); + var second = Queue(contract, protocol, Command.Create(descriptor)); Assert.IsTrue(await first.MoveNextAsync()); await first.DisposeAsync(); @@ -217,23 +249,26 @@ public async Task SuccessorProgressesAfterAbandonment() } [ConnectionCreatingTestMethod] - public async Task GracefulStopDrainsHeldResultAndFaultsConsumer() + [DynamicData(nameof(Implementations))] + public async Task GracefulStopDrainsHeldResultAndFaultsConsumer(IConsumerDrivenFlowContract contract) { - var protocol = await PgTestPool.NewIsolatedAsync(options => - options.HeartbeatInterval = TimeSpan.FromMilliseconds(20)); - var descriptor = await Prepare(protocol, + var protocol = await PgTestPool.NewIsolatedAsync(); + var descriptor = await Prepare(contract, protocol, "select generate_series(1, 1000)", "contract_graceful"); - var results = Queue(protocol, Command.Create(descriptor)); + var results = Queue(contract, protocol, Command.Create(descriptor)); Assert.IsTrue(await results.MoveNextAsync()); - await protocol.CompleteAsync().WaitAsync(TimeSpan.FromSeconds(5)); + var complete = protocol.CompleteAsync(); + await protocol.Heartbeat(TimeSpan.Zero); + await complete; await Assert.ThrowsAsync( async () => await results.MoveNextAsync()); await results.DisposeAsync(); } [ConnectionCreatingTestMethod(connections: 2)] - public async Task BackendTermination_IsCollateral() + [DynamicData(nameof(Implementations))] + public async Task BackendTermination_IsCollateral(IConsumerDrivenFlowContract contract) { await using var protocols = await PgTestPool.NewIsolatedProtocolsAsync(2); var killer = protocols.Items[1]; @@ -244,10 +279,10 @@ public async Task BackendTermination_IsCollateral() async Task Terminate() { await using var victim = await PgTestPool.NewIsolatedAsync(); - var pid = await ReadBackendPid(victim); - var descriptor = await Prepare(victim, "select pg_sleep(10)", + var pid = await ReadBackendPid(contract, victim); + var descriptor = await Prepare(contract, victim, "select pg_sleep(10)", "contract_terminate_command"); - var results = Queue(victim, Command.Create(descriptor)); + var results = Queue(contract, victim, Command.Create(descriptor)); var pending = results.MoveNextAsync(); Assert.IsFalse(pending.IsCompleted); await PgTestPool.RunAsync(killer, $"select pg_terminate_backend({pid})"); @@ -259,10 +294,11 @@ async Task Terminate() } [ConnectionCreatingTestMethod] - public async Task TornTrailingWrite_RecoversWire() + [DynamicData(nameof(Implementations))] + public async Task TornTrailingWrite_RecoversWire(IConsumerDrivenFlowContract contract) { await using var protocol = await PgTestPool.NewIsolatedAsync(); - var results = Queue(protocol, TornStreamedBind()); + var results = Queue(contract, protocol, TornStreamedBind()); Exception? observed = null; try @@ -292,13 +328,37 @@ public async Task TornTrailingWrite_RecoversWire() await PgTestPool.RunAsync(protocol, "select 42::int4"); } + [ConnectionCreatingTestMethod] + [DynamicData(nameof(ReusableImplementations))] + public async Task ResetAfterCompletion_StartsIndependentTenure( + IReusableConsumerDrivenFlowContract contract) + { + await using var protocol = await PgTestPool.NewIsolatedAsync(); + var flow = contract.CreateReusable(async: true, Command.Create("select 1")); + + for (var tenure = 0; tenure < 2; tenure++) + { + if (tenure > 0) + contract.Reset(flow, async: true, Command.Create("select 2")); + protocol.Queue(flow); + var results = contract.GetAsyncEnumerator(flow); + Assert.IsTrue(await results.MoveNextAsync(), + $"tenure {tenure} must publish its result"); + await results.Current.DisposeAsync(); + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + await flow.WaitForComplete(); + } + } + static Task NewCancelableProtocolAsync() => PgTestPool.NewIsolatedAsync(options => options.CancelSender = PgTestPool.CreateCancelSender(PgTestPool.NewOptions())); - static async Task ReadBackendPid(PgClientProtocol protocol) + static async Task ReadBackendPid( + IConsumerDrivenFlowContract contract, PgClientProtocol protocol) { - var results = Queue(protocol, Command.Create("select pg_backend_pid()")); + var results = Queue(contract, protocol, Command.Create("select pg_backend_pid()")); var pid = 0; while (await results.MoveNextAsync()) { @@ -357,11 +417,13 @@ public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); } - readonly struct Results(CommandFlow.Enumerator inner) : IAsyncDisposable + readonly struct Results( + IConsumerDrivenFlowContract contract, + IAsyncEnumerator inner) : IAsyncDisposable { internal CommandResult Current => inner.Current; internal ValueTask MoveNextAsync(CancellationToken cancellationToken = default) - => inner.MoveNextAsync(cancellationToken); + => contract.MoveNextAsync(inner, cancellationToken); public ValueTask DisposeAsync() => inner.DisposeAsync(); } diff --git a/Slon.Tests/Pg/FlowAuthoring/README.md b/Slon.Tests/Pg/FlowAuthoring/README.md index 209f345..05193a7 100644 --- a/Slon.Tests/Pg/FlowAuthoring/README.md +++ b/Slon.Tests/Pg/FlowAuthoring/README.md @@ -10,8 +10,10 @@ result-shape tests remain with the implementation. task and needs no external consumer to reach RFQ. - `IConsumerDrivenFlowContract` applies when an external sync or async consumer owns result advancement and can abandon that ownership. +- `IReusableConsumerDrivenFlowContract` adds the requirement that a completed + instance can be reset for an independent tenure. -A flow exposing both kinds of entry point should register an adapter for every +A flow exposing multiple kinds of entry point should register an adapter for every profile it supports. Profiles are separate interfaces so an implementation cannot silently opt out of individual required scenarios with capability flags. @@ -50,6 +52,7 @@ lifecycle as independent axes. In particular: autonomous flow; - an async consumer never runs on the pipeline executor strand; - release occurs before completion makes a flow reusable. +- a reset tenure does not retain terminal or consumer state from its predecessor. Contract scenarios must use explicit scheduler gates, in-memory transports, or `FakeTimeProvider` when ordering matters. Do not use wall-clock delays or From a84def159a6f6033b15c21ebfa81083450ea32da Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:50:38 +0200 Subject: [PATCH 101/136] Restore prepared ADO command fast paths --- Slon/Ado/AdoCommandFactory.cs | 88 ++++++++++++++++++----- Slon/Ado/AdoCommandFlowFactory.cs | 111 ++++++++++++++++++++++++++++++ Slon/Ado/TrackedCommand.cs | 3 +- 3 files changed, 184 insertions(+), 18 deletions(-) diff --git a/Slon/Ado/AdoCommandFactory.cs b/Slon/Ado/AdoCommandFactory.cs index a89bcb3..0d96a2f 100644 --- a/Slon/Ado/AdoCommandFactory.cs +++ b/Slon/Ado/AdoCommandFactory.cs @@ -1,6 +1,7 @@ using System.Data; using System.Data.Common; using System.Diagnostics; +using System.Runtime.CompilerServices; using Slon.Pg; using Slon.Pg.Serialization; using Slon.Pg.Types; @@ -22,6 +23,49 @@ interface IAdoCommand static class AdoCommandFactory { + [MethodImpl(MethodImplOptions.NoInlining)] + internal static Command CreatePreparedCommandWithParameters( + in TCommand command, TrackedCommand tracked, in CommandDescriptor descriptor, + SlonParameters? commandParameters, bool enableErrorBarriers, CommandBehavior behavior, + DbParameterCollection? dbParameters, TimeSpan timeout, + PgSerializerOptions serializerOptions, ParameterWriter parameterWriter) + where TCommand : IAdoCommand + { + if (tracked.Kind is not TrackedCommandKind.Command + && dbParameters is not null && commandParameters is { Count: > 0 }) + { + throw new InvalidOperationException( + "Execution parameters cannot be combined with parameters stored on the command."); + } + + dbParameters ??= commandParameters; + if (dbParameters is not null and not SlonParameters) + { + throw new ArgumentException( + $"Execution parameters must be a {nameof(SlonParameters)} instance.", nameof(dbParameters)); + } + if ((dbParameters?.Count ?? 0) != descriptor.ParameterTypes.Count) + { + throw new InvalidOperationException( + $"Prepared command expects {descriptor.ParameterTypes.Count} parameters, " + + $"received {dbParameters?.Count ?? 0}."); + } + + var parameters = dbParameters is { Count: > 0 } + ? ResolveNonEmptyParameters( + (SlonParameters)dbParameters, serializerOptions, descriptor.ParameterTypes, + allowUnspecified: false, parameterWriter) + : default; + return new Command + { + Descriptor = descriptor, + DescribeOnly = behavior.HasFlag(CommandBehavior.SchemaOnly), + WithSync = enableErrorBarriers || command.AppendErrorBarrier, + Parameters = parameters, + Timeout = timeout + }; + } + public static (Command, TrackerResult) CreateCommand(in TCommand command, bool allowAutoPreparation, bool enableErrorBarriers, CommandBehavior behavior, in TrackerContext trackerContext, DbParameterCollection? dbParameters, TimeSpan timeout, @@ -66,23 +110,9 @@ public static (Command, TrackerResult) CreateCommand(in TCommand comma { if (serializerOptions is null) ThrowHelper.ThrowInvalidOperation("ADO parameter serialization requires serializer options."); - using var preparedTypes = preparedParameterTypes.GetEnumerator(); - var parameterIndex = 0; - foreach (var kv in slonParameters!.GetStructEnumerator()) - { - if (kv.Key != SlonParameters.PositionalName) - { - throw new NotSupportedException( - "Named parameters are not yet supported; they require client-side SQL parsing."); - } - - var currentParameterIndex = parameterIndex++; - var preparedType = preparedTypes.MoveNext() ? preparedTypes.Current : (PgTypeId?)null; - slonParameters.GetOrResolveTypeInfo( - currentParameterIndex, serializerOptions, preparedType, allowUnspecified: preparing); - } - - parameters = new(slonParameters!, + parameters = ResolveNonEmptyParameters( + slonParameters!, serializerOptions, preparedParameterTypes, + allowUnspecified: preparing, parameterWriter ?? throw new InvalidOperationException( "ADO parameter serialization requires a parameter writer.")); parameterTypes = new(parameters); @@ -113,4 +143,28 @@ public static (Command, TrackerResult) CreateCommand(in TCommand comma Timeout = timeout }, trackerResult); } + + static ParameterSource ResolveNonEmptyParameters( + SlonParameters parameters, PgSerializerOptions serializerOptions, + ParameterTypeList preparedParameterTypes, bool allowUnspecified, + ParameterWriter parameterWriter) + { + using var preparedTypes = preparedParameterTypes.GetEnumerator(); + var parameterIndex = 0; + foreach (var parameter in parameters.GetStructEnumerator()) + { + if (parameter.Key != SlonParameters.PositionalName) + { + throw new NotSupportedException( + "Named parameters are not yet supported; they require client-side SQL parsing."); + } + + var currentParameterIndex = parameterIndex++; + var preparedType = preparedTypes.MoveNext() ? preparedTypes.Current : (PgTypeId?)null; + parameters.GetOrResolveTypeInfo( + currentParameterIndex, serializerOptions, preparedType, allowUnspecified); + } + + return new(parameters, parameterWriter); + } } diff --git a/Slon/Ado/AdoCommandFlowFactory.cs b/Slon/Ado/AdoCommandFlowFactory.cs index c7ccc12..a8e2636 100644 --- a/Slon/Ado/AdoCommandFlowFactory.cs +++ b/Slon/Ado/AdoCommandFlowFactory.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Data; using System.Data.Common; +using System.Runtime.CompilerServices; using Slon.Pg; using Slon.Pg.Protocol.Flows; @@ -33,6 +34,26 @@ public AdoCommandFlowOptions Create( ThrowHelper.ThrowArgumentException(nameof(parametersSpan), "The number of parameter collections must match the number of commands."); var pendingPrefix = connection?.TakePendingTransactionStatement(); + if (commands.Length is 1 && explicitlyPrepared && !preparing && pendingPrefix is null) + return CreatePreparedSingle( + parametersSpan.IsEmpty ? null : parametersSpan[0], + behavior, enableErrorBarriers, + timeout, pgConnection, pendingTimeout); + + return CreateGeneral( + parametersSpan, behavior, explicitlyPrepared, allowAutoPreparation, enableErrorBarriers, + timeout, connection, pgConnection, pendingTimeout, preparing, + pendingPrefix, indexParameters); + } + + AdoCommandFlowOptions CreateGeneral( + ReadOnlySpan parametersSpan, CommandBehavior behavior, + bool explicitlyPrepared, bool allowAutoPreparation, bool enableErrorBarriers, TimeSpan timeout, + SlonConnection? connection, PgConnection? pgConnection, + TimeSpan? pendingTimeout, bool preparing, string? pendingPrefix, + bool indexParameters) + { + var commands = _commands; var commandOffset = pendingPrefix is null ? 0 : 1; var commandCount = commands.Length + commandOffset; var commandArray = commandCount > 1 ? ArrayPool.Shared.Rent(commandCount) : null; @@ -149,6 +170,96 @@ public AdoCommandFlowOptions Create( } } + [MethodImpl(MethodImplOptions.NoInlining)] + AdoCommandFlowOptions CreatePreparedSingle( + DbParameterCollection? parameters, CommandBehavior behavior, + bool enableErrorBarriers, TimeSpan timeout, + PgConnection? pgConnection, TimeSpan? pendingTimeout) + { + ref var adoCommand = ref _commands[0]; + var tracked = adoCommand.Tracked + ?? throw new InvalidOperationException( + "The explicitly prepared command has no tracked command."); + if (!tracked.TryGetPreparedDescriptor(out var descriptor)) + throw new InvalidOperationException( + "The explicitly prepared command has no prepared descriptor."); + var commandParameters = adoCommand.Parameters; + var command = parameters is null && commandParameters is null + && descriptor.ParameterTypes.Count is 0 + ? new Command + { + Descriptor = descriptor, + DescribeOnly = behavior.HasFlag(CommandBehavior.SchemaOnly), + WithSync = enableErrorBarriers || adoCommand.AppendErrorBarrier, + Parameters = default, + Timeout = timeout + } + : AdoCommandFactory.CreatePreparedCommandWithParameters( + adoCommand, tracked, descriptor, commandParameters, enableErrorBarriers, + behavior, parameters, timeout, + dependencies.SerializerOptions, dependencies.ParameterWriter); + + Action? resultAction = null; + object? resultActionState = null; + if (pgConnection is not null) + { + var status = pgConnection.GetTrackedStatus(tracked); + if (status is TrackedStatus.Tracked) + { + resultAction = AdoCommandResultObserver.AttachPrepared; + resultActionState = pgConnection; + } + else + { + var preparation = ResolvePreparedSingleSlow( + adoCommand, command, tracked, pgConnection, status); + command = preparation.Command; + resultAction = preparation.ResultAction; + resultActionState = preparation.ResultActionState; + } + } + + return new() + { + ResultObserver = resultAction, + ResultObserverState = resultActionState, + Commands = new(command), + PendingTimeout = pendingTimeout + }; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static PreparationResolution ResolvePreparedSingleSlow( + TCommand adoCommand, Command command, TrackedCommand tracked, + PgConnection connection, TrackedStatus status) + { + var parameterTypes = command.Descriptor.ParameterTypes; + if (status is TrackedStatus.Preparing || !connection.TryBeginPreparing(tracked)) + { + return new(command with + { + Descriptor = CommandDescriptor.Create( + adoCommand.CommandText, parameterTypes, default) + }, null, null); + } + + try + { + command = command with + { + Descriptor = CommandDescriptor.Create( + adoCommand.CommandText, parameterTypes, tracked.CommandName) + }; + return new(command, AdoCommandResultObserver.ObservePreparing, + (connection, tracked, (SlonBatchCommand?)null)); + } + catch + { + connection.RemoveTracked(tracked); + throw; + } + } + static PreparationResolution ResolvePreparation( TCommand adoCommand, Command command, TrackedCommand tracked, SlonBatchCommand? batchCommand, PgConnection connection, diff --git a/Slon/Ado/TrackedCommand.cs b/Slon/Ado/TrackedCommand.cs index f4b6dca..28ff087 100644 --- a/Slon/Ado/TrackedCommand.cs +++ b/Slon/Ado/TrackedCommand.cs @@ -65,7 +65,8 @@ internal bool TryGetPreparedDescriptor(out CommandDescriptor descriptor) // Sample across this thread's lookups: hot commands remain recent without a clock read and // shared timestamp write on every execution. - if ((++accessSampleCounter & AccessSampleMask) is 0) + if (Kind is TrackedCommandKind.Auto + && (++accessSampleCounter & AccessSampleMask) is 0) Volatile.Write(ref _lastAccessedTicks, Environment.TickCount64); descriptor = state.Descriptor; return true; From 2fcfd6fbf5f7793c38e64f3e2c7b02e336212011 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:50:39 +0200 Subject: [PATCH 102/136] Restore ADO reader completion fast paths --- Slon.Tests/DataReaderTests.cs | 20 +++++ Slon/Ado/AdoCommandFlow.cs | 2 + Slon/Pg/CommandResult.cs | 3 + Slon/PublicAPI.Unshipped.txt | 1 - Slon/SlonDataReader.cs | 135 ++++++++++++++++++++++++++++------ 5 files changed, 137 insertions(+), 24 deletions(-) diff --git a/Slon.Tests/DataReaderTests.cs b/Slon.Tests/DataReaderTests.cs index 7292e6a..1d951dd 100644 --- a/Slon.Tests/DataReaderTests.cs +++ b/Slon.Tests/DataReaderTests.cs @@ -181,6 +181,26 @@ public async Task ExecuteNonQuery_NonDataModifying_IsZero() Assert.AreEqual(-1, await AdoTestPool.ExecuteNonQueryAsync("SELECT generate_series(1, 10)"), "SELECT 10 rows"); } + [TestMethod] + public async Task ExecuteReader_ReportsRowsAffectedAfterSkippingNonRowResult() + { + await using var connection = await AdoTestPool.OpenConnectionAsync(); + await using (var setup = connection.CreateCommand( + "CREATE TEMP TABLE reader_records_affected (value int)")) + _ = await setup.ExecuteNonQueryAsync(); + await using (var insert = connection.CreateCommand( + "INSERT INTO reader_records_affected VALUES (1), (2)")) + _ = await insert.ExecuteNonQueryAsync(); + + await using var command = connection.CreateCommand( + "UPDATE reader_records_affected SET value = value + 1"); + await using var reader = await command.ExecuteReaderAsync(); + + Assert.IsFalse(await reader.ReadAsync()); + Assert.AreEqual(2, reader.RecordsAffected); + Assert.AreEqual(2L, reader.LongRecordsAffected); + } + [TestMethod] public async Task BatchExecuteNonQuery_SumsAllCommandResults() { diff --git a/Slon/Ado/AdoCommandFlow.cs b/Slon/Ado/AdoCommandFlow.cs index 626f9bc..542fcc4 100644 --- a/Slon/Ado/AdoCommandFlow.cs +++ b/Slon/Ado/AdoCommandFlow.cs @@ -250,6 +250,8 @@ void IValueTaskSource.OnCompleted(Action continuation, object? state, s internal Enumerator(AdoCommandExecutionFlow flow, CancellationToken cancellationToken) => (_flow, _cancellationToken) = (flow, cancellationToken); + internal bool IsDefault => _flow is null; + public bool MoveNext() => _flow?.Core.MoveNext() ?? false; public ValueTask MoveNextAsync() => MoveNextAsync(_cancellationToken); public ValueTask MoveNextAsync(CancellationToken cancellationToken) diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index 8bf6791..fb58784 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -209,6 +209,9 @@ public long RecordsAffected } } + internal long BatchRecordsAffected + => _commandCompleteMessage?.BatchRecordsAffected ?? -1; + internal void CompleteNonQuery(BackendMessage message) { if (message.Header.Type is PgTypes.BackendType.DataRow) diff --git a/Slon/PublicAPI.Unshipped.txt b/Slon/PublicAPI.Unshipped.txt index 8622b53..d292969 100644 --- a/Slon/PublicAPI.Unshipped.txt +++ b/Slon/PublicAPI.Unshipped.txt @@ -366,7 +366,6 @@ Slon.SlonConnectionInitializerContext.Connection.get -> Slon.SlonConnection! Slon.SlonDataReader Slon.SlonDataReader.GetBytes(int ordinal) -> byte[]! Slon.SlonDataReader.GetColumnSchema() -> System.Collections.ObjectModel.ReadOnlyCollection! -Slon.SlonDataReader.GetData(int ordinal) -> Slon.SlonDataReader! Slon.SlonDataReader.LongRecordsAffected.get -> long Slon.SlonDataSource Slon.SlonDataSource.CreateBatch() -> Slon.SlonBatch! diff --git a/Slon/SlonDataReader.cs b/Slon/SlonDataReader.cs index 71847a1..14eaf39 100644 --- a/Slon/SlonDataReader.cs +++ b/Slon/SlonDataReader.cs @@ -16,6 +16,26 @@ public sealed partial class SlonDataReader { const CommandBehavior EnumerateCommandResultsBehavior = (CommandBehavior)int.MinValue; + interface IReadPolicy + { + static abstract ValueTask MoveNext( + ref CommandResult.RowEnumerator rows, CancellationToken cancellationToken); + } + + readonly struct DefaultReadPolicy : IReadPolicy + { + public static ValueTask MoveNext( + ref CommandResult.RowEnumerator rows, CancellationToken cancellationToken) + => rows.MoveNextAsync(); + } + + readonly struct CancelableReadPolicy : IReadPolicy + { + public static ValueTask MoveNext( + ref CommandResult.RowEnumerator rows, CancellationToken cancellationToken) + => rows.MoveNextAsync(cancellationToken); + } + int _state; ReaderState State { @@ -126,6 +146,8 @@ internal static SlonDataReader Create(CommandBehavior behavior, AdoCommandExecut } } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] internal static async ValueTask CreateAsync(CommandBehavior behavior, ValueTask flowTask, PgSerializerOptions serializerOptions, CancellationToken cancellationToken = default, @@ -196,7 +218,8 @@ static void ApplyCompletion(CommandResult current, ref long? recordsAffected) { current.TryGetCommandComplete(out _); if (current.Error is null) - recordsAffected += current.RecordsAffected; + AccumulateRecordsAffected( + ref recordsAffected, current.BatchRecordsAffected); } } @@ -312,10 +335,18 @@ void ApplyCompletion(CommandResult current) { SurfaceCompletion(current); if (current.Error is null) - _recordsAffected += current.RecordsAffected; + AccumulateRecordsAffected( + ref _recordsAffected, current.BatchRecordsAffected); _currentCompletion = ResultCompletionState.Applied; } + static void AccumulateRecordsAffected(ref long? total, long current) + { + if (current < 0) + return; + total = total is { } existing ? checked(existing + current) : current; + } + bool ReadCore() { try @@ -339,33 +370,78 @@ bool ReadCore() } } - async Task ReadAsyncCore(CancellationToken cancellationToken) + Task ReadAsyncCore(CancellationToken cancellationToken) + where TPolicy : struct, IReadPolicy { try { Debug.Assert(_singleRowBehavior && _remainingResults is 0 || !_singleRowBehavior); - bool hasRow; if (_rowPresence is RowPresence.Prefetched) { _rowPresence = RowPresence.Present; - return true; + return Task.FromResult(true); } - else if (_singleRowBehavior && _rowPresence is RowPresence.Present) - { + + bool hasRow; + if (_singleRowBehavior && _rowPresence is RowPresence.Present) hasRow = false; - } else { - hasRow = await _rowEnumerator.MoveNextAsync(cancellationToken) - .ConfigureAwait(false); + var moveNext = TPolicy.MoveNext(ref _rowEnumerator, cancellationToken); + if (!moveNext.IsCompletedSuccessfully) + return AwaitMoveNext(this, moveNext); + hasRow = moveNext.GetAwaiter().GetResult(); } - if (hasRow) - return ProcessReadResult(hasRow: true); + return CompleteRead(this, hasRow); + } + catch (Exception ex) + { + return Task.FromException(AdoException.Project(ex)); + } + } + + static Task CompleteRead(SlonDataReader reader, bool hasRow) + { + if (hasRow) + return Task.FromResult(reader.ProcessReadResult(hasRow: true)); - if (Current is { IsComplete: false } current) + if (reader.Current is { IsComplete: false } current) + { + var completion = current.CompleteAsync(); + if (!completion.IsCompletedSuccessfully) + return AwaitCompletion(reader, completion); + completion.GetAwaiter().GetResult(); + } + return Task.FromResult(reader.ProcessReadResult(hasRow: false)); + } + + static async Task AwaitMoveNext( + SlonDataReader reader, ValueTask moveNext) + { + try + { + var hasRow = await moveNext.ConfigureAwait(false); + if (hasRow) + return reader.ProcessReadResult(hasRow: true); + if (reader.Current is { IsComplete: false } current) await current.CompleteAsync().ConfigureAwait(false); - return ProcessReadResult(hasRow: false); + return reader.ProcessReadResult(hasRow: false); + } + catch (Exception ex) + { + AdoException.Throw(ex); + return default; + } + } + + static async Task AwaitCompletion( + SlonDataReader reader, ValueTask completion) + { + try + { + await completion.ConfigureAwait(false); + return reader.ProcessReadResult(hasRow: false); } catch (Exception ex) { @@ -765,6 +841,8 @@ public override Task NextResultAsync(CancellationToken cancellationToken) { if (GetExceptionIfClosedOrDisposed() is { } exception) return Task.FromException(exception); + if (_remainingResults is 0 && _currentCompletion is ResultCompletionState.Applied) + return Task.FromResult(false); return NextResultAsyncCore(cancellationToken); } @@ -783,7 +861,9 @@ public override Task ReadAsync(CancellationToken cancellationToken) if (GetExceptionIfClosedOrDisposed() is { } exception) return Task.FromException(exception); - return ReadAsyncCore(cancellationToken); + return cancellationToken.CanBeCanceled + ? ReadAsyncCore(cancellationToken) + : ReadAsyncCore(default); } /// @@ -843,16 +923,9 @@ public override Task IsDBNullAsync(int ordinal, CancellationToken cancella : IsDBNullAsyncCore(ordinal, cancellationToken); } - /// Returns a nested data reader for the requested column. - /// The zero-based column ordinal. - /// Nested data readers are not supported. - /// A data reader. - public new SlonDataReader GetData(int ordinal) - => throw new NotSupportedException("Nested data readers are not supported."); - /// protected override DbDataReader GetDbDataReader(int ordinal) - => GetData(ordinal); + => throw new NotSupportedException("Nested data readers are not supported."); /// Reads the complete field at the specified ordinal as a byte array. /// The zero-based column ordinal. @@ -1014,6 +1087,14 @@ protected override void Dispose(bool disposing) return; State = ReaderState.Disposed; + if (_remainingResults is 0 + && _currentCompletion is ResultCompletionState.Applied + && _enumerator.IsDefault + && _connectionToClose is null) + { + Reset(); + return; + } CloseCore(resetForReuse: true); } @@ -1024,6 +1105,14 @@ public override ValueTask DisposeAsync() return new(); State = ReaderState.Disposed; + if (_remainingResults is 0 + && _currentCompletion is ResultCompletionState.Applied + && _enumerator.IsDefault + && _connectionToClose is null) + { + Reset(); + return default; + } return CloseAsyncCore(resetForReuse: true); } } From a359fc9e81409723705f231aac9d8e5eafbea6e7 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:50:39 +0200 Subject: [PATCH 103/136] Reuse resolved connections during reader creation --- Slon/Ado/AdoBatchCore.cs | 73 +++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/Slon/Ado/AdoBatchCore.cs b/Slon/Ado/AdoBatchCore.cs index 665549e..e907fe4 100644 --- a/Slon/Ado/AdoBatchCore.cs +++ b/Slon/Ado/AdoBatchCore.cs @@ -180,16 +180,24 @@ internal AdoCommandFlowOptions CreateAdoCommandFlowOptions( } SlonDataSource.PgDbDependencies GetDependencies() + => GetDependencies(out _); + + SlonDataSource.PgDbDependencies GetDependencies( + out SlonConnection? connection) { - TryGetDataSource(out var dataSource, out var connection); + TryGetDataSource(out var dataSource, out connection); connection ??= dataSource is null ? ThrowConnectionNotInitialized() : null; return (dataSource ?? connection!.DbDataSource).GetDbDependencies(); } ValueTask GetDependenciesAsync( CancellationToken cancellationToken) + => GetDependenciesAsync(cancellationToken, out _); + + ValueTask GetDependenciesAsync( + CancellationToken cancellationToken, out SlonConnection? connection) { - TryGetDataSource(out var dataSource, out var connection); + TryGetDataSource(out var dataSource, out connection); connection ??= dataSource is null ? ThrowConnectionNotInitialized() : null; return (dataSource ?? connection!.DbDataSource).GetDbDependenciesAsync(cancellationToken); } @@ -434,16 +442,10 @@ public SlonDataReader ExecuteReader(DbParameterCollection? parameters, CommandBe SlonDataReader ExecuteReaderCore(DbParameterCollection? parameters, CommandBehavior behavior) { - var dependencies = GetDependencies(); + var dependencies = GetDependencies(out var connection); return SlonDataReader.Create(behavior, Enqueue(parameters, behavior, dependencies), - dependencies.SerializerOptions, GetConnectionToClose(behavior)); - } - - SlonConnection? GetConnectionToClose(CommandBehavior behavior) - { - if (!HasCloseConnection(behavior) || TryGetDataSource(out _, out var connection)) - return null; - return connection; + dependencies.SerializerOptions, + HasCloseConnection(behavior) ? connection : null); } public ValueTask ExecuteDbReaderAsync(DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken = default) @@ -477,13 +479,14 @@ static ValueTask ExecuteReaderAsyncCore( var activity = core.StartActivity(); try { - var connectionToClose = core.GetConnectionToClose(behavior); - var dependenciesTask = core.GetDependenciesAsync(cancellationToken); + var closeConnection = core.HasCloseConnection(behavior); + var dependenciesTask = core.GetDependenciesAsync( + cancellationToken, out var connection); return dependenciesTask.IsCompletedSuccessfully ? BeginReaderCreation(fieldRef, parameters, behavior, cancellationToken, - connectionToClose, dependenciesTask.Result, activity) + connection, closeConnection, dependenciesTask.Result, activity) : AwaitDependenciesAndCreateReaderAsync(fieldRef, parameters, behavior, - cancellationToken, connectionToClose, dependenciesTask, activity); + cancellationToken, connection, closeConnection, dependenciesTask, activity); } catch (Exception ex) { @@ -494,7 +497,8 @@ static ValueTask ExecuteReaderAsyncCore( static ValueTask BeginReaderCreation( FieldRef> fieldRef, DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken, - SlonConnection? connectionToClose, SlonDataSource.PgDbDependencies dependencies, + SlonConnection? connection, bool closeConnection, + SlonDataSource.PgDbDependencies dependencies, Activity? activity) where TReader : DbDataReader { @@ -502,10 +506,13 @@ static ValueTask BeginReaderCreation( { return SlonDataReader.CreateAsync(behavior, fieldRef.Invoke().EnqueueAsync(parameters, behavior, dependencies, cancellationToken), - dependencies.SerializerOptions, cancellationToken, connectionToClose, activity); + dependencies.SerializerOptions, cancellationToken, + closeConnection ? connection : null, activity); } catch (Exception ex) { + if (closeConnection && connection is not null) + return FailAndCloseReaderCreation(connection, activity, ex); return FailReaderCreation(activity, ex); } } @@ -513,7 +520,7 @@ static ValueTask BeginReaderCreation( static async ValueTask AwaitDependenciesAndCreateReaderAsync( FieldRef> fieldRef, DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken, - SlonConnection? connectionToClose, + SlonConnection? connection, bool closeConnection, ValueTask dependenciesTask, Activity? activity) where TReader : DbDataReader { @@ -531,7 +538,7 @@ static async ValueTask AwaitDependenciesAndCreateReaderAsync( } return await BeginReaderCreation(fieldRef, parameters, behavior, cancellationToken, - connectionToClose, dependencies, activity).ConfigureAwait(false); + connection, closeConnection, dependencies, activity).ConfigureAwait(false); } static ValueTask FailReaderCreation(Activity? activity, Exception exception) @@ -542,7 +549,35 @@ static ValueTask FailReaderCreation(Activity? activity, Except return ValueTask.FromException(AdoException.Project(exception)); } + static async ValueTask FailAndCloseReaderCreation( + SlonConnection connection, Activity? activity, Exception exception) + where TReader : DbDataReader + { + try + { + await connection.CloseAsync().ConfigureAwait(false); + } + catch (Exception cleanupException) + { + exception = cleanupException; + } + SlonTracing.RecordException(activity, exception); + activity?.Dispose(); + AdoException.Throw(exception); + return default!; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] Activity? StartActivity() + { + if (!SlonTracing.ShouldStart) + return null; + + return StartActivityCore(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + Activity? StartActivityCore() { TryGetDataSource(out var dataSource, out var connection); return dataSource is null && connection is null From dfd3c9b5a2940307e0ae1199e6c2b8450b1ee97c Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:52:16 +0200 Subject: [PATCH 104/136] Derive ADO flow lifetime ownership from its binding --- Slon/Ado/AdoBatchCore.cs | 12 ++++-------- Slon/Ado/AdoCommandFlow.cs | 14 +++++++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Slon/Ado/AdoBatchCore.cs b/Slon/Ado/AdoBatchCore.cs index e907fe4..5e508c5 100644 --- a/Slon/Ado/AdoBatchCore.cs +++ b/Slon/Ado/AdoBatchCore.cs @@ -214,9 +214,7 @@ AdoCommandExecutionFlow Enqueue(DbParameterCollection? parameters, CommandBehavi async: false, (IAdoCommandExecutionOwner)_fieldRef.Instance, parameters, behavior, dependencies, connection: null, pendingTimeout, preparing, _commands.Count, - _explicitlyPrepared && _fieldRef.Instance is SlonCommand - ? null - : (IAdoCommandExecutionOwner)_fieldRef.Instance), + ownsLifetime: !(_explicitlyPrepared && _fieldRef.Instance is SlonCommand)), pendingTimeout); } @@ -224,7 +222,7 @@ AdoCommandExecutionFlow Enqueue(DbParameterCollection? parameters, CommandBehavi return connection.Enqueue(new AdoCommandExecutionFlow( async: false, (IAdoCommandExecutionOwner)_fieldRef.Instance, parameters, behavior, dependencies, connection, PendingTimeout, preparing, - _commands.Count, (IAdoCommandExecutionOwner)_fieldRef.Instance)); + _commands.Count, ownsLifetime: true)); } ValueTask EnqueueAsync(DbParameterCollection? parameters, @@ -240,9 +238,7 @@ ValueTask EnqueueAsync(DbParameterCollection? parameter async: true, (IAdoCommandExecutionOwner)_fieldRef.Instance, parameters, behavior, dependencies, connection: null, pendingTimeout, preparing, _commands.Count, - _explicitlyPrepared && _fieldRef.Instance is SlonCommand - ? null - : (IAdoCommandExecutionOwner)_fieldRef.Instance), + ownsLifetime: !(_explicitlyPrepared && _fieldRef.Instance is SlonCommand)), pendingTimeout, cancellationToken); } @@ -250,7 +246,7 @@ ValueTask EnqueueAsync(DbParameterCollection? parameter return connection.EnqueueAsync(new AdoCommandExecutionFlow( async: true, (IAdoCommandExecutionOwner)_fieldRef.Instance, parameters, behavior, dependencies, connection, PendingTimeout, preparing, - _commands.Count, (IAdoCommandExecutionOwner)_fieldRef.Instance), cancellationToken); + _commands.Count, ownsLifetime: true), cancellationToken); } [DoesNotReturn] diff --git a/Slon/Ado/AdoCommandFlow.cs b/Slon/Ado/AdoCommandFlow.cs index 542fcc4..5bf49fb 100644 --- a/Slon/Ado/AdoCommandFlow.cs +++ b/Slon/Ado/AdoCommandFlow.cs @@ -116,7 +116,7 @@ sealed class AdoCommandExecutionFlow : PgClientFlow, IValueTaskSource, IVa readonly SlonConnection? _connection; readonly bool _preparing; readonly int _commandCount; - IAdoCommandExecutionOwner? _lifetimeOwner; + int _lifetimePending; Action? _resultObserver; object? _resultObserverState; CommandExecutionState _state; @@ -126,7 +126,7 @@ internal AdoCommandExecutionFlow( DbParameterCollection? parameters, CommandBehavior behavior, SlonDataSource.PgDbDependencies dependencies, SlonConnection? connection, TimeSpan? pendingTimeout, bool preparing, int commandCount, - IAdoCommandExecutionOwner? lifetimeOwner) + bool ownsLifetime) : base(supportsDeferredFlush: true) { _bindingOwner = bindingOwner; @@ -136,7 +136,7 @@ internal AdoCommandExecutionFlow( _connection = connection; _preparing = preparing; _commandCount = commandCount; - _lifetimeOwner = lifetimeOwner; + _lifetimePending = ownsLifetime ? 1 : 0; _state.CommandIndex = -1; _state.EnableActivationTimeout = true; _state.WaitForDrainOnDispose = true; @@ -145,7 +145,8 @@ internal AdoCommandExecutionFlow( if (!async) _state.HandoffEvent = new(false); SetObserver(AdoCommandExecutionObserver.Instance, null); - lifetimeOwner?.OnFlowStarted(this); + if (ownsLifetime) + bindingOwner.OnFlowStarted(this); } internal override bool DefersSyncHandoff => true; @@ -192,7 +193,10 @@ void ObserveResult(CommandResult result) => _resultObserver?.Invoke(result, _resultObserverState); internal void CompleteLifetime(Exception? exception) - => Interlocked.Exchange(ref _lifetimeOwner, null)?.OnFlowCompleting(this, exception); + { + if (Interlocked.Exchange(ref _lifetimePending, 0) is not 0) + _bindingOwner.OnFlowCompleting(this, exception); + } internal override void Bind(PgClientFlowBindingContext? context) { From edffeb765d6f8b452181a09b7eefdc1d5d990af9 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 15:59:02 +0200 Subject: [PATCH 105/136] Name the shared command flow core directly --- Slon/Ado/AdoCommandFlow.cs | 2 +- Slon/Pg/Protocol/Flows/CommandFlow.cs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Slon/Ado/AdoCommandFlow.cs b/Slon/Ado/AdoCommandFlow.cs index 5bf49fb..5368717 100644 --- a/Slon/Ado/AdoCommandFlow.cs +++ b/Slon/Ado/AdoCommandFlow.cs @@ -174,7 +174,7 @@ public Enumerator GetAsyncEnumerator(CancellationToken cancellationToken = defau return new(this, cancellationToken); } - CommandExecutionCore Core => new(new(this)); + CommandFlowCore Core => new(new(this)); internal ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) => Core.ConsumeNonQueryAsync(cancellationToken); diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 8b5afd2..d374cdb 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -185,7 +185,7 @@ internal bool WaitForDrainOnDispose set => _state.WaitForDrainOnDispose = value; } - CommandExecutionCore Core => new(new(this)); + CommandFlowCore Core => new(new(this)); internal ValueTask ConsumeNonQueryAsync(CancellationToken cancellationToken = default) => Core.ConsumeNonQueryAsync(cancellationToken); @@ -245,7 +245,7 @@ internal interface ICommandExecutionFlowOps void OnDiscarded(); } -readonly struct CommandExecutionCore(TOps ops) +readonly struct CommandFlowCore(TOps ops) where TOps : struct, ICommandExecutionFlowOps { const int PhaseInitial = 0; @@ -331,7 +331,7 @@ internal ValueTask ExecuteAuto(PgClientFlow.Context context) OnActivationSettled(onExecutorStrand: true); else activation.UnsafeOnCompleted(static state => - new CommandExecutionCore(TOps.Create((PgClientFlow)state!)) + new CommandFlowCore(TOps.Create((PgClientFlow)state!)) .OnActivationSettled(onExecutorStrand: !((PgClientFlow)state!).ActivationWasDispatched), _ops.Flow); return new(new FlowTasks(writeTask, new ValueTask((IValueTaskSource)_ops.Flow, _state.PipelineTaskSource.Version))); @@ -1036,7 +1036,7 @@ bool TryTakeOverDrain() // Decoder takeover does not imply consumer abandonment. Explicit cancellation and // graceful close also drain autonomously while retaining their consumer semantics. ThreadPool.UnsafeQueueUserWorkItem(static state => - _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ = new CommandFlowCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), _ops.Flow); return true; } @@ -1107,7 +1107,7 @@ void HandleReadTimeout(TimeoutException exception) RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); ThreadPool.UnsafeQueueUserWorkItem(static state => - _ = new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ = new CommandFlowCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), _ops.Flow); } @@ -1306,13 +1306,13 @@ void RegisterCancellation(CancellationToken callerToken) } if (callerToken.CanBeCanceled && cancellation.CallerRegistration == default) cancellation.CallerRegistration = callerToken.UnsafeRegister(static (state, token) - => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( + => new CommandFlowCore(TOps.Create((PgClientFlow)state!)).RequestCancel( token, CommandExecutionCancellationScope.CurrentWindow), _ops.Flow); } if (_state.FlowToken.CanBeCanceled && _state.FlowRegistration == default) _state.FlowRegistration = _state.FlowToken.UnsafeRegister(static (state, token) - => new CommandExecutionCore(TOps.Create((PgClientFlow)state!)).RequestCancel( + => new CommandFlowCore(TOps.Create((PgClientFlow)state!)).RequestCancel( token, CommandExecutionCancellationScope.RemainingFlow), _ops.Flow); } From 1c73b8ac54bed60acc307c119a9a7896455524df Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 16:10:41 +0200 Subject: [PATCH 106/136] Replace delegated field references with owner contracts --- .../AdoCommandFlowFactoryBenchmark.cs | 2 +- Slon.Tests/Pg/SyncFlowHandoffTests.cs | 13 ++-- Slon/Ado/AdoBatchCore.Preparation.cs | 42 +++++------ Slon/Ado/AdoBatchCore.cs | 70 +++++++++++-------- Slon/Ado/AdoCommandFlow.cs | 2 +- Slon/Pg/Protocol/Flows/CommandFlow.cs | 7 +- .../Flows/FlowCallerInteractionCore.cs | 14 ++-- Slon/Runtime/CompilerServices/FieldRef.cs | 31 -------- Slon/Runtime/CompilerServices/IFieldRef.cs | 9 +++ Slon/SlonBatch.cs | 25 ++++--- Slon/SlonBatchCommands.cs | 11 ++- Slon/SlonCommand.cs | 22 +++--- 12 files changed, 125 insertions(+), 123 deletions(-) delete mode 100644 Slon/Runtime/CompilerServices/FieldRef.cs create mode 100644 Slon/Runtime/CompilerServices/IFieldRef.cs diff --git a/Slon.Benchmark/AdoCommandFlowFactoryBenchmark.cs b/Slon.Benchmark/AdoCommandFlowFactoryBenchmark.cs index f4f50c3..68941dc 100644 --- a/Slon.Benchmark/AdoCommandFlowFactoryBenchmark.cs +++ b/Slon.Benchmark/AdoCommandFlowFactoryBenchmark.cs @@ -144,5 +144,5 @@ public async Task Cleanup() } [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_batchCore")] - static extern ref AdoBatchCore GetBatchCore(SlonBatch batch); + static extern ref AdoBatchCore GetBatchCore(SlonBatch batch); } diff --git a/Slon.Tests/Pg/SyncFlowHandoffTests.cs b/Slon.Tests/Pg/SyncFlowHandoffTests.cs index cbeb7db..571d40d 100644 --- a/Slon.Tests/Pg/SyncFlowHandoffTests.cs +++ b/Slon.Tests/Pg/SyncFlowHandoffTests.cs @@ -34,7 +34,11 @@ sealed class WakeHolder internal FlowCallerInteractionCore Core; } - static ref FlowCallerInteractionCore GetWakeCore(WakeHolder holder) => ref holder.Core; + readonly struct WakeRef(WakeHolder holder) + : IFieldRef> + { + public ref FlowCallerInteractionCore GetField() => ref holder.Core; + } [ConnectionCreatingTestMethod] public async Task PairedAsyncAndSync_NoSharedPromiseCollision() @@ -73,12 +77,7 @@ public async Task DedicatedWakeResumesOutsideTheThreadPool() static async Task Suspend(WakeHolder holder, TaskCompletionSource resumedOnThreadPool) { - FieldRef> fieldRef; - unsafe - { - fieldRef = FieldRef>.Create(&GetWakeCore, holder); - } - await holder.Core.YieldToCaller(fieldRef); + await holder.Core.YieldToCaller(new WakeRef(holder)); resumedOnThreadPool.SetResult(Thread.CurrentThread.IsThreadPoolThread); } } diff --git a/Slon/Ado/AdoBatchCore.Preparation.cs b/Slon/Ado/AdoBatchCore.Preparation.cs index d5285d6..cdd7ded 100644 --- a/Slon/Ado/AdoBatchCore.Preparation.cs +++ b/Slon/Ado/AdoBatchCore.Preparation.cs @@ -6,7 +6,9 @@ namespace Slon; -partial struct AdoBatchCore where TCommand : IAdoCommand +partial struct AdoBatchCore + where TCommand : IAdoCommand + where TFieldRef : struct, IAdoBatchCoreRef { public void Prepare(DbParameterCollection? parameters) { @@ -64,10 +66,10 @@ public ValueTask PrepareAsync(DbParameterCollection? parameters, CancellationToken cancellationToken = default) => PrepareAsyncProjected(_fieldRef, parameters, cancellationToken); - static async ValueTask PrepareAsyncProjected(FieldRef> fieldRef, + static async ValueTask PrepareAsyncProjected(TFieldRef fieldRef, DbParameterCollection? parameters, CancellationToken cancellationToken) { - using var activity = fieldRef.Invoke().StartActivity(); + using var activity = fieldRef.GetField().StartActivity(); try { await PrepareAsyncCore(fieldRef, parameters, cancellationToken).ConfigureAwait(false); @@ -80,17 +82,17 @@ static async ValueTask PrepareAsyncProjected(FieldRef> fi } // Async instance methods on structs copy this, so the state machine resolves the live core - // through its stable field reference instead. - static async ValueTask PrepareAsyncCore(FieldRef> fieldRef, + // through its stable fieldRef reference instead. + static async ValueTask PrepareAsyncCore(TFieldRef fieldRef, DbParameterCollection? parameters, CancellationToken cancellationToken) { var operation = Preparation.Begin(fieldRef); AdoCommandExecutionFlow.Enumerator enumerator = default; try { - var dependencies = await fieldRef.Invoke().GetDependenciesAsync(cancellationToken) + var dependencies = await fieldRef.GetField().GetDependenciesAsync(cancellationToken) .ConfigureAwait(false); - var flow = await fieldRef.Invoke().EnqueueAsync(parameters, CommandBehavior.SchemaOnly, + var flow = await fieldRef.GetField().EnqueueAsync(parameters, CommandBehavior.SchemaOnly, dependencies, cancellationToken, preparing: true).ConfigureAwait(false); enumerator = flow.GetAsyncEnumerator(cancellationToken); for (var i = 0; i < operation.CommandCount; i++) @@ -124,12 +126,12 @@ static async ValueTask PrepareAsyncCore(FieldRef> fieldRe struct Preparation { - readonly FieldRef> _fieldRef; + readonly TFieldRef _fieldRef; readonly SlonDataSource? _dataSource; readonly SlonConnection? _connection; List? _exceptions; - Preparation(FieldRef> fieldRef, + Preparation(TFieldRef fieldRef, SlonDataSource? dataSource, SlonConnection? connection) { _fieldRef = fieldRef; @@ -137,16 +139,16 @@ struct Preparation _connection = connection; } - internal static Preparation Begin(FieldRef> fieldRef) + internal static Preparation Begin(TFieldRef fieldRef) { - ref var core = ref fieldRef.Invoke(); + ref var core = ref fieldRef.GetField(); core.ThrowIfDisposedOrReadOnly(); core.TryGetDataSource(out var dataSource, out var connection); core._explicitlyPrepared = true; return new(fieldRef, dataSource, connection); } - internal int CommandCount => _fieldRef.Invoke()._commands.Count; + internal int CommandCount => _fieldRef.GetField()._commands.Count; internal void Observe(CommandResult result) { @@ -169,34 +171,34 @@ internal void ThrowIfFailed() internal void Commit() { - foreach (ref var command in _fieldRef.Invoke()._commands.AsSpan()) + foreach (ref var command in _fieldRef.GetField()._commands.AsSpan()) command.MakeReadOnly(); } internal void Rollback() - => _fieldRef.Invoke()._explicitlyPrepared = false; + => _fieldRef.GetField()._explicitlyPrepared = false; internal void ReleaseFailedPreparation() { - if (_fieldRef.Invoke()._explicitlyPrepared) + if (_fieldRef.GetField()._explicitlyPrepared) return; if (_connection is not null) - _connection.UnprepareOwned(async: false, _fieldRef.Instance).GetAwaiter().GetResult(); + _connection.UnprepareOwned(async: false, _fieldRef.Owner).GetAwaiter().GetResult(); else if (_dataSource is not null) _ = _dataSource.ReleaseOwnedPreparedCommand( - _fieldRef.Instance, awaitable: false); + _fieldRef.Owner, awaitable: false); } internal ValueTask ReleaseFailedPreparationAsync() { - if (_fieldRef.Invoke()._explicitlyPrepared) + if (_fieldRef.GetField()._explicitlyPrepared) return default; if (_connection is not null) - return _connection.UnprepareOwned(async: true, _fieldRef.Instance); + return _connection.UnprepareOwned(async: true, _fieldRef.Owner); return _dataSource?.ReleaseOwnedPreparedCommand( - _fieldRef.Instance, awaitable: true) ?? default; + _fieldRef.Owner, awaitable: true) ?? default; } } } diff --git a/Slon/Ado/AdoBatchCore.cs b/Slon/Ado/AdoBatchCore.cs index 5e508c5..a7f41eb 100644 --- a/Slon/Ado/AdoBatchCore.cs +++ b/Slon/Ado/AdoBatchCore.cs @@ -9,10 +9,20 @@ namespace Slon; +interface IAdoBatchCoreRef + : IFieldRef> + where TCommand : IAdoCommand + where TFieldRef : struct, IAdoBatchCoreRef +{ + IAdoCommandExecutionOwner Owner { get; } +} + // Shared between DbBatch and DbCommand -partial struct AdoBatchCore where TCommand : IAdoCommand +partial struct AdoBatchCore + where TCommand : IAdoCommand + where TFieldRef : struct, IAdoBatchCoreRef { - readonly FieldRef> _fieldRef; + readonly TFieldRef _fieldRef; object _dataSourceOrConnection; bool _disposed; bool _explicitlyPrepared; @@ -23,19 +33,19 @@ partial struct AdoBatchCore where TCommand : IAdoCommand AdoCommandExecutionFlow? _activeFlow; AdoCommandList _commands; - public AdoBatchCore(FieldRef> fieldRef) + public AdoBatchCore(TFieldRef fieldRef) { _dataSourceOrConnection = null!; _fieldRef = fieldRef; } - public AdoBatchCore(SlonConnection connection, FieldRef> fieldRef) + public AdoBatchCore(TFieldRef fieldRef, SlonConnection connection) { _dataSourceOrConnection = connection; _fieldRef = fieldRef; } - public AdoBatchCore(SlonDataSource dataSource, FieldRef> fieldRef) + public AdoBatchCore(TFieldRef fieldRef, SlonDataSource dataSource) { _dataSourceOrConnection = dataSource; _fieldRef = fieldRef; @@ -113,7 +123,7 @@ public void ThrowIfDisposedOrReadOnly() public void ThrowIfDisposed() { if (_disposed) - Throw(_fieldRef.Instance); + Throw(_fieldRef.Owner); static void Throw(object instance) => throw new ObjectDisposedException(instance.GetType().Name); } @@ -172,7 +182,7 @@ internal AdoCommandFlowOptions CreateAdoCommandFlowOptions( PgConnection? pgConnection = null, TimeSpan? pendingTimeout = null, bool preparing = false) { var factory = new AdoCommandFlowFactory( - _fieldRef.Instance, _commands.AsSpan(), dependencies); + _fieldRef.Owner, _commands.AsSpan(), dependencies); return factory.Create( parametersSpan, behavior, _explicitlyPrepared, _allowAutoPreparation, _enableErrorBarriers, Timeout, @@ -211,16 +221,16 @@ AdoCommandExecutionFlow Enqueue(DbParameterCollection? parameters, CommandBehavi var pendingTimeout = PendingTimeout; return dataSource.EnqueueCommands( new AdoCommandExecutionFlow( - async: false, (IAdoCommandExecutionOwner)_fieldRef.Instance, + async: false, _fieldRef.Owner, parameters, behavior, dependencies, connection: null, pendingTimeout, preparing, _commands.Count, - ownsLifetime: !(_explicitlyPrepared && _fieldRef.Instance is SlonCommand)), + ownsLifetime: !(_explicitlyPrepared && _fieldRef.Owner is SlonCommand)), pendingTimeout); } connection ??= ThrowConnectionNotInitialized(); return connection.Enqueue(new AdoCommandExecutionFlow( - async: false, (IAdoCommandExecutionOwner)_fieldRef.Instance, + async: false, _fieldRef.Owner, parameters, behavior, dependencies, connection, PendingTimeout, preparing, _commands.Count, ownsLifetime: true)); } @@ -235,16 +245,16 @@ ValueTask EnqueueAsync(DbParameterCollection? parameter var pendingTimeout = PendingTimeout; return dataSource.EnqueueCommandsAsync( new AdoCommandExecutionFlow( - async: true, (IAdoCommandExecutionOwner)_fieldRef.Instance, + async: true, _fieldRef.Owner, parameters, behavior, dependencies, connection: null, pendingTimeout, preparing, _commands.Count, - ownsLifetime: !(_explicitlyPrepared && _fieldRef.Instance is SlonCommand)), + ownsLifetime: !(_explicitlyPrepared && _fieldRef.Owner is SlonCommand)), pendingTimeout, cancellationToken); } connection ??= ThrowConnectionNotInitialized(); return connection.EnqueueAsync(new AdoCommandExecutionFlow( - async: true, (IAdoCommandExecutionOwner)_fieldRef.Instance, + async: true, _fieldRef.Owner, parameters, behavior, dependencies, connection, PendingTimeout, preparing, _commands.Count, ownsLifetime: true), cancellationToken); } @@ -289,9 +299,9 @@ int ExecuteNonQueryCore(DbParameterCollection? parameters) return checked((int)recordsAffected); } - static async ValueTask ExecuteNonQueryAsyncCore(FieldRef> fieldRef, DbParameterCollection? parameters, CancellationToken cancellationToken) + static async ValueTask ExecuteNonQueryAsyncCore(TFieldRef fieldRef, DbParameterCollection? parameters, CancellationToken cancellationToken) { - ref var thisRef = ref fieldRef.Invoke(); + ref var thisRef = ref fieldRef.GetField(); using var activity = thisRef.StartActivity(); try { @@ -299,7 +309,7 @@ static async ValueTask ExecuteNonQueryAsyncCore(FieldRef ExecuteNonQueryAsync(DbParameterCollection? parameters, Ca return null; } - static async ValueTask ExecuteScalarAsyncCore(FieldRef> fieldRef, DbParameterCollection? parameters, CancellationToken cancellationToken) + static async ValueTask ExecuteScalarAsyncCore(TFieldRef fieldRef, DbParameterCollection? parameters, CancellationToken cancellationToken) { - ref var thisRef = ref fieldRef.Invoke(); + ref var thisRef = ref fieldRef.GetField(); using var activity = thisRef.StartActivity(); AdoCommandExecutionFlow.Enumerator enumerator = default; try @@ -366,7 +376,7 @@ public ValueTask ExecuteNonQueryAsync(DbParameterCollection? parameters, Ca var dependencies = await thisRef.GetDependenciesAsync(cancellationToken) .ConfigureAwait(false); var fieldReader = new PgSerializerFieldReader(dependencies.SerializerOptions); - enumerator = (await fieldRef.Invoke().EnqueueAsync(parameters, CommandBehavior.Default, + enumerator = (await fieldRef.GetField().EnqueueAsync(parameters, CommandBehavior.Default, dependencies, cancellationToken).ConfigureAwait(false)) .GetAsyncEnumerator(cancellationToken); while (await enumerator.MoveNextAsync().ConfigureAwait(false)) @@ -447,7 +457,7 @@ SlonDataReader ExecuteReaderCore(DbParameterCollection? parameters, CommandBehav public ValueTask ExecuteDbReaderAsync(DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken = default) { if (_disposed) - return ValueTask.FromException(new ObjectDisposedException(_fieldRef.Instance.GetType().Name)); + return ValueTask.FromException(new ObjectDisposedException(_fieldRef.Owner.GetType().Name)); if (cancellationToken.IsCancellationRequested) return ValueTask.FromCanceled(cancellationToken); @@ -458,7 +468,7 @@ public ValueTask ExecuteDbReaderAsync(DbParameterCollection? param public ValueTask ExecuteReaderAsync(DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken = default) { if (_disposed) - return ValueTask.FromException(new ObjectDisposedException(_fieldRef.Instance.GetType().Name)); + return ValueTask.FromException(new ObjectDisposedException(_fieldRef.Owner.GetType().Name)); if (cancellationToken.IsCancellationRequested) return ValueTask.FromCanceled(cancellationToken); @@ -467,11 +477,11 @@ public ValueTask ExecuteReaderAsync(DbParameterCollection? param } static ValueTask ExecuteReaderAsyncCore( - FieldRef> fieldRef, DbParameterCollection? parameters, + TFieldRef fieldRef, DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken) where TReader : DbDataReader { - ref var core = ref fieldRef.Invoke(); + ref var core = ref fieldRef.GetField(); var activity = core.StartActivity(); try { @@ -491,7 +501,7 @@ static ValueTask ExecuteReaderAsyncCore( } static ValueTask BeginReaderCreation( - FieldRef> fieldRef, DbParameterCollection? parameters, + TFieldRef fieldRef, DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken, SlonConnection? connection, bool closeConnection, SlonDataSource.PgDbDependencies dependencies, @@ -501,7 +511,7 @@ static ValueTask BeginReaderCreation( try { return SlonDataReader.CreateAsync(behavior, - fieldRef.Invoke().EnqueueAsync(parameters, behavior, dependencies, cancellationToken), + fieldRef.GetField().EnqueueAsync(parameters, behavior, dependencies, cancellationToken), dependencies.SerializerOptions, cancellationToken, closeConnection ? connection : null, activity); } @@ -514,7 +524,7 @@ static ValueTask BeginReaderCreation( } static async ValueTask AwaitDependenciesAndCreateReaderAsync( - FieldRef> fieldRef, DbParameterCollection? parameters, + TFieldRef fieldRef, DbParameterCollection? parameters, CommandBehavior behavior, CancellationToken cancellationToken, SlonConnection? connection, bool closeConnection, ValueTask dependenciesTask, Activity? activity) @@ -601,12 +611,12 @@ public void Dispose() return; if (TryGetDataSource(out var dataSource, out var connection)) { - _ = dataSource.ReleaseOwnedPreparedCommand(_fieldRef.Instance, awaitable: false); + _ = dataSource.ReleaseOwnedPreparedCommand(_fieldRef.Owner, awaitable: false); return; } if (connection is not null) - connection.UnprepareOwned(async: false, _fieldRef.Instance).GetAwaiter().GetResult(); + connection.UnprepareOwned(async: false, _fieldRef.Owner).GetAwaiter().GetResult(); } public ValueTask DisposeAsync() @@ -618,9 +628,9 @@ public ValueTask DisposeAsync() if (!_explicitlyPrepared) return new(); if (TryGetDataSource(out var dataSource, out var connection)) - return dataSource.ReleaseOwnedPreparedCommand(_fieldRef.Instance, awaitable: true); + return dataSource.ReleaseOwnedPreparedCommand(_fieldRef.Owner, awaitable: true); - return connection?.UnprepareOwned(async: true, _fieldRef.Instance) ?? default; + return connection?.UnprepareOwned(async: true, _fieldRef.Owner) ?? default; } public void Cancel() diff --git a/Slon/Ado/AdoCommandFlow.cs b/Slon/Ado/AdoCommandFlow.cs index 5368717..4afa18b 100644 --- a/Slon/Ado/AdoCommandFlow.cs +++ b/Slon/Ado/AdoCommandFlow.cs @@ -218,7 +218,7 @@ readonly struct Ops(AdoCommandExecutionFlow owner) : ICommandExecutionFlowOps new((AdoCommandExecutionFlow)flow); public PgClientFlow Flow => _owner; - public ref CommandExecutionState State => ref _owner._state; + public ref CommandExecutionState GetField() => ref _owner._state; public bool IsAsync { get => _owner.IsAsync; diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index d374cdb..1cbb7a3 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -209,7 +209,7 @@ readonly struct Ops(CommandFlow owner) : ICommandExecutionFlowOps public static Ops Create(PgClientFlow flow) => new((CommandFlow)flow); public PgClientFlow Flow => _owner; - public ref CommandExecutionState State => ref _owner._state; + public ref CommandExecutionState GetField() => ref _owner._state; public bool IsAsync { get => _owner.IsAsync; @@ -230,12 +230,11 @@ public void OnDiscarded() } } -internal interface ICommandExecutionFlowOps +internal interface ICommandExecutionFlowOps : IFieldRef where TSelf : struct, ICommandExecutionFlowOps { static abstract TSelf Create(PgClientFlow flow); PgClientFlow Flow { get; } - ref CommandExecutionState State { get; } bool IsAsync { get; set; } bool IsAsyncAtDispatch { get; } bool HasSuccessfulActivation { get; } @@ -255,7 +254,7 @@ readonly struct CommandFlowCore(TOps ops) const int PhaseCompleted = 4; readonly TOps _ops = ops; - ref CommandExecutionState _state => ref _ops.State; + ref CommandExecutionState _state => ref _ops.GetField(); internal bool IsResultReady => Volatile.Read(ref _state.Phase) is PhaseResultReady; bool IsSinglePublishedCommand => _state.Commands.Count is 1 diff --git a/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs b/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs index 7d23835..5328487 100644 --- a/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs +++ b/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs @@ -180,7 +180,9 @@ public void SignalProgress() bool ConsumeProgress() => Interlocked.Exchange(ref _progressSignaled, 0) != 0; - public CallerHandoffAwaitable YieldToCaller(FieldRef> fieldRef) + public CallerHandoffAwaitable YieldToCaller(TFieldRef fieldRef) + where TFieldRef : struct, + IFieldRef> => new(fieldRef); public void Reset() @@ -193,11 +195,13 @@ public void Reset() Volatile.Write(ref _gateCompletionClaim, 0); } - public readonly struct CallerHandoffAwaitable(FieldRef> fieldRef) + public readonly struct CallerHandoffAwaitable(TFieldRef fieldRef) + where TFieldRef : struct, + IFieldRef> { public Awaiter GetAwaiter() => new(fieldRef); - public readonly struct Awaiter(FieldRef> fieldRef) : ICriticalNotifyCompletion + public readonly struct Awaiter(TFieldRef fieldRef) : ICriticalNotifyCompletion { public bool IsCompleted => false; @@ -206,14 +210,14 @@ public void GetResult() // Surface a pending cancellation set by FaultBodyWait. The gate source is // only completed when cancellation fires. In the normal sync-flow handoff path // it stays Pending and we just return. - var gate = fieldRef.Invoke()._gate; + var gate = fieldRef.GetField()._gate; if (gate.GetStatus(gate.Version) != System.Threading.Tasks.Sources.ValueTaskSourceStatus.Pending) gate.GetResult(gate.Version); } public void OnCompleted(Action continuation) { - ref var field = ref fieldRef.Invoke(); + ref var field = ref fieldRef.GetField(); var waitEvent = field.GetWaitEvent(); if (!ReferenceEquals(waitEvent.HandoffContinuation, continuation)) Volatile.Write(ref waitEvent.HandoffContinuation, continuation); diff --git a/Slon/Runtime/CompilerServices/FieldRef.cs b/Slon/Runtime/CompilerServices/FieldRef.cs deleted file mode 100644 index 6e72c79..0000000 --- a/Slon/Runtime/CompilerServices/FieldRef.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Runtime.CompilerServices; - -namespace Slon.Runtime.CompilerServices; - -// This is safe against unloading as long as instance is the same type as the getter function is defined on. -// There is unfortunately no static type safety to guarantee a user creates one correctly. -readonly struct FieldRef -{ - readonly unsafe delegate* _getter; - readonly object _instance; - - unsafe FieldRef(delegate* getter, object instance) - { - _getter = getter; - _instance = instance; - } - - public object Instance => _instance; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref T Invoke() - { - unsafe - { - return ref _getter(_instance); - } - } - - public static unsafe FieldRef Create(delegate* getter, TInstance instance) where TInstance : class - => new((delegate*)getter, instance); -} diff --git a/Slon/Runtime/CompilerServices/IFieldRef.cs b/Slon/Runtime/CompilerServices/IFieldRef.cs new file mode 100644 index 0000000..9ef1423 --- /dev/null +++ b/Slon/Runtime/CompilerServices/IFieldRef.cs @@ -0,0 +1,9 @@ +namespace Slon.Runtime.CompilerServices; + +// Provides reusable, allocation-free access to a field retained by another object or struct. +// Generic consumers preserve the concrete implementation type so the ref-returning call can inline. +interface IFieldRef + where TOwner : struct, IFieldRef +{ + ref T GetField(); +} diff --git a/Slon/SlonBatch.cs b/Slon/SlonBatch.cs index 54bee1c..c382d26 100644 --- a/Slon/SlonBatch.cs +++ b/Slon/SlonBatch.cs @@ -9,28 +9,36 @@ namespace Slon; /// public sealed partial class SlonBatch { - AdoBatchCore _batchCore; + internal readonly struct BatchCoreRef(SlonBatch owner) + : IAdoBatchCoreRef + { + public IAdoCommandExecutionOwner Owner => owner; + public ref AdoBatchCore GetField() + => ref owner._batchCore; + } + + AdoBatchCore _batchCore; SlonBatchCommands? _batchCommands; - unsafe SlonBatch(SlonConnection? connection, SlonDataSource? dataSource) + SlonBatch(SlonConnection? connection, SlonDataSource? dataSource) { - var fieldRef = FieldRef>.Create(&GetBatchCore, this); if (connection is not null) { - _batchCore = new(connection, fieldRef); + _batchCore = new(new(this), connection); _batchCore.Timeout = connection.DefaultCommandTimeout; } else if (dataSource is not null) { - _batchCore = new(dataSource, fieldRef); + _batchCore = new(new(this), dataSource); _batchCore.Timeout = dataSource.DefaultCommandTimeout; } else - _batchCore = new(fieldRef); + _batchCore = new(new(this)); } - unsafe SlonBatchCommands CreateBatchCommandCollection() - => new(FieldRef>.Create(&GetBatchCore, this)); + SlonBatchCommands CreateBatchCommandCollection() => new(this); + + internal ref AdoBatchCore BatchCore => ref _batchCore; internal void OnFlowStarted(AdoCommandExecutionFlow flow) => _batchCore.OnFlowStarted(flow); @@ -53,7 +61,6 @@ void IAdoCommandExecutionOwner.OnFlowCompleting( AdoCommandExecutionFlow flow, Exception? exception) => OnFlowCompleting(flow, exception); - static ref AdoBatchCore GetBatchCore(SlonBatch instance) => ref instance._batchCore; } // Public surface & ADO.NET diff --git a/Slon/SlonBatchCommands.cs b/Slon/SlonBatchCommands.cs index 880368a..2c88c33 100644 --- a/Slon/SlonBatchCommands.cs +++ b/Slon/SlonBatchCommands.cs @@ -1,22 +1,21 @@ using System.Data.Common; -using Slon.Runtime.CompilerServices; namespace Slon; /// public sealed class SlonBatchCommands : DbBatchCommandCollection, IList { - readonly FieldRef> _batchRef; + readonly SlonBatch _batch; - internal SlonBatchCommands(FieldRef> batchRef) => _batchRef = batchRef; + internal SlonBatchCommands(SlonBatch batch) => _batch = batch; - ref AdoCommandList List => ref _batchRef.Invoke().Commands; + ref AdoCommandList List => ref _batch.BatchCore.Commands; /// public override int Count => List.Count; /// - public override bool IsReadOnly => _batchRef.Invoke().IsReadOnly; + public override bool IsReadOnly => _batch.BatchCore.IsReadOnly; /// IEnumerator IEnumerable.GetEnumerator() @@ -142,6 +141,6 @@ static SlonBatchCommand ThrowInvalidCastException(DbBatchCommand? value) => void ThrowIfReadOnly() { - _batchRef.Invoke().ThrowIfDisposedOrReadOnly(); + _batch.BatchCore.ThrowIfDisposedOrReadOnly(); } } diff --git a/Slon/SlonCommand.cs b/Slon/SlonCommand.cs index 1986a95..edbbd94 100644 --- a/Slon/SlonCommand.cs +++ b/Slon/SlonCommand.cs @@ -11,7 +11,15 @@ namespace Slon; /// public sealed partial class SlonCommand { - AdoBatchCore _batchCore; + readonly struct BatchCoreRef(SlonCommand owner) + : IAdoBatchCoreRef + { + public IAdoCommandExecutionOwner Owner => owner; + public ref AdoBatchCore GetField() + => ref owner._batchCore; + } + + AdoBatchCore _batchCore; // Supporting state for implicit batching through SQL parsing. string _overallCommandText; @@ -19,28 +27,24 @@ public sealed partial class SlonCommand SlonParameters? _overallParameterCollection; bool _isOverallStateDirty; - internal unsafe SlonCommand(SlonConnection? connection, SlonDataSource? dataSource, string? commandText) + internal SlonCommand(SlonConnection? connection, SlonDataSource? dataSource, string? commandText) { GC.SuppressFinalize(this); _isOverallStateDirty = true; _overallCommandText = commandText ?? string.Empty; _overallCommandType = CommandType.Text; - var fieldRef = FieldRef>.Create(&GetBatchCore, this); if (connection is not null) { - _batchCore = new(connection, fieldRef); + _batchCore = new(new(this), connection); _batchCore.Timeout = connection.DefaultCommandTimeout; } else if (dataSource is not null) { - _batchCore = new(dataSource, fieldRef); + _batchCore = new(new(this), dataSource); _batchCore.Timeout = dataSource.DefaultCommandTimeout; } else - _batchCore = new(fieldRef); - - // ReSharper disable once AddressOfMarshalByRefObject - static ref AdoBatchCore GetBatchCore(SlonCommand instance) => ref instance._batchCore; + _batchCore = new(new(this)); } void SetupCommands() From a3f8c6d41004de874fdb1340c6f0147e35131511 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 16:11:29 +0200 Subject: [PATCH 107/136] Update the backend publication benchmark for cursors --- Slon.Benchmark/BackendMessagePublicationBenchmark.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Slon.Benchmark/BackendMessagePublicationBenchmark.cs b/Slon.Benchmark/BackendMessagePublicationBenchmark.cs index 9446d76..59f0480 100644 --- a/Slon.Benchmark/BackendMessagePublicationBenchmark.cs +++ b/Slon.Benchmark/BackendMessagePublicationBenchmark.cs @@ -15,8 +15,8 @@ public class BackendMessagePublicationBenchmark [Benchmark] public int PublishBufferedMessages() { - _context.RetireCurrentBatch(); - _context.SetBatch(new BackendMessageBatch(new ReadOnlySequence(_messages))); + _context.RetireCursor(); + _context.SetCursor(new BackendMessageCursor(new ReadOnlySequence(_messages))); var count = 0; while (_context.TryMoveNext()) count++; From 519f4b5f4cd532583c9f6e21135381cb9e0d78df Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 16:48:40 +0200 Subject: [PATCH 108/136] Expose buffered command result collection --- Slon.Tests/Pg/CommandResultCollectTests.cs | 38 ++++++++++-- .../Pg/CommandResultEnumerationTests.cs | 12 ++-- .../Pg/PublicCommandFlowSurfaceTests.cs | 23 +++++++ Slon/Pg/CommandResult.cs | 61 ++++++++++++++----- .../Flows/CommandFlow.MessageEnumerator.cs | 12 ++-- Slon/Pg/Protocol/PgClientProtocol.cs | 2 +- Slon/Pg/Protocol/PgDecoder.cs | 38 ++++++------ Slon/Pg/Protocol/ProtocolReadPipe.cs | 40 ++++++------ Slon/Pg/Row.cs | 2 +- 9 files changed, 155 insertions(+), 73 deletions(-) diff --git a/Slon.Tests/Pg/CommandResultCollectTests.cs b/Slon.Tests/Pg/CommandResultCollectTests.cs index 6931208..62c3b5d 100644 --- a/Slon.Tests/Pg/CommandResultCollectTests.cs +++ b/Slon.Tests/Pg/CommandResultCollectTests.cs @@ -16,7 +16,7 @@ public async Task CollectsValueRowsAndLeavesWireReusable() Command.Create("select generate_series(1, 100)"))).GetAsyncEnumerator(); Assert.IsTrue(await results.MoveNextAsync()); - await results.Current.CollectRowsAsync(values, + await results.Current.CollectAsync(values, static (items, row) => items.Add(row.GetInt32(0))); Assert.IsFalse(await results.MoveNextAsync()); await results.DisposeAsync(); @@ -35,7 +35,7 @@ public async Task BuffersAStreamingRowBeforeCallingCollector() Command.Create("select 42, repeat('x', 100000)"))).GetAsyncEnumerator(); Assert.IsTrue(await results.MoveNextAsync()); - await results.Current.CollectRowsAsync(values, + await results.Current.CollectAsync(values, static (items, row) => items.Add((row.GetInt32(0), row.GetValue(1)))); Assert.IsFalse(await results.MoveNextAsync()); await results.DisposeAsync(); @@ -46,6 +46,35 @@ await results.Current.CollectRowsAsync(values, await PgTestPool.RunAsync(protocol, "select 1"); } + [ConnectionCreatingTestMethod] + public async Task ResultBufferingRetainsCollectedRowsPastTheBufferingBarrier() + { + await using var protocol = await PgTestPool.NewIsolatedAsync( + options => options.DataRowStreamingThreshold = 1); + var values = new List<(int Id, ReadOnlyMemory Message)>(); + var results = protocol.Queue(new CommandFlow(async: true, Command.Create( + "select i, i::text || repeat('x', 20000) " + + "from generate_series(1, 3) as i"))).GetAsyncEnumerator(); + + Assert.IsTrue(await results.MoveNextAsync()); + results.Current.EnableResultBuffering(); + await results.Current.CollectAsync(values, + static (rows, row) => rows.Add(( + row.GetInt32(0), row.BorrowFieldMemory(1)))); + + Assert.AreEqual(3, values.Count); + for (var i = 0; i < values.Count; i++) + { + Assert.AreEqual(i + 1, values[i].Id); + Assert.AreEqual((i + 1) + new string('x', 20000), + System.Text.Encoding.UTF8.GetString(values[i].Message.Span)); + } + + Assert.IsFalse(await results.MoveNextAsync()); + await results.DisposeAsync(); + await PgTestPool.RunAsync(protocol, "select 1"); + } + [ConnectionCreatingTestMethod] public async Task CollectorFailureDrainsBeforeRethrowing() { @@ -56,7 +85,7 @@ public async Task CollectorFailureDrainsBeforeRethrowing() Assert.IsTrue(await results.MoveNextAsync()); var exception = await Assert.ThrowsExactlyAsync(async () => - await results.Current.CollectRowsAsync(0, (_, _) => + await results.Current.CollectAsync(0, (_, _) => { callbacks++; throw new InvalidOperationException("collector failure"); @@ -80,10 +109,9 @@ public async Task CannotCollectAfterRowEnumerationStarted() var rows = results.Current.GetAsyncEnumerator(); Assert.IsTrue(await rows.MoveNextAsync()); await Assert.ThrowsExactlyAsync(async () => - await results.Current.CollectRowsAsync(0, static (_, _) => { })); + await results.Current.CollectAsync(0, static (_, _) => { })); await rows.DisposeAsync(); Assert.IsFalse(await results.MoveNextAsync()); await results.DisposeAsync(); } } - diff --git a/Slon.Tests/Pg/CommandResultEnumerationTests.cs b/Slon.Tests/Pg/CommandResultEnumerationTests.cs index 1a8ab21..f8ebde9 100644 --- a/Slon.Tests/Pg/CommandResultEnumerationTests.cs +++ b/Slon.Tests/Pg/CommandResultEnumerationTests.cs @@ -9,7 +9,7 @@ namespace Slon.Tests.Pg; public class CommandResultEnumerationTests { [ConnectionCreatingTestMethod] - public async Task ResultSetBuffering_CannotBeginAfterRowEnumeration() + public async Task ResultBuffering_CannotBeginAfterRowEnumeration() { await using var protocol = await PgTestPool.NewIsolatedAsync(); var flow = protocol.Queue(new CommandFlow( @@ -21,7 +21,7 @@ public async Task ResultSetBuffering_CannotBeginAfterRowEnumeration() var rows = result.GetAsyncEnumerator(); Assert.IsTrue(await rows.MoveNextAsync()); Assert.ThrowsExactly( - result.EnableResultSetBuffering); + result.EnableResultBuffering); await rows.DisposeAsync(); await results.DisposeAsync(); @@ -40,7 +40,7 @@ public async Task ContiguousFieldMemory_RemainsValidAcrossExtendedBatches() try { Assert.IsTrue(await results.MoveNextAsync()); - results.Current.EnableResultSetBuffering(); + results.Current.EnableResultBuffering(); var rows = results.Current.GetAsyncEnumerator(); while (await rows.MoveNextAsync()) { @@ -74,7 +74,7 @@ public async Task ContiguousFieldMemory_BuffersStreamingRowsIntoTheRetainedBatch try { Assert.IsTrue(await results.MoveNextAsync()); - results.Current.EnableResultSetBuffering(); + results.Current.EnableResultBuffering(); var rows = results.Current.GetAsyncEnumerator(); while (await rows.MoveNextAsync()) values.Add(rows.Current.BorrowFieldMemory(1)); @@ -94,7 +94,7 @@ public async Task ContiguousFieldMemory_BuffersStreamingRowsIntoTheRetainedBatch } [ConnectionCreatingTestMethod] - public async Task ResultSetBuffering_AbandonmentReleasesTheReadGrant() + public async Task ResultBuffering_AbandonmentReleasesTheReadGrant() { await using var protocol = await PgTestPool.NewIsolatedAsync(); var flow = protocol.Queue(new CommandFlow(async: true, Command.Create( @@ -102,7 +102,7 @@ public async Task ResultSetBuffering_AbandonmentReleasesTheReadGrant() var results = flow.GetAsyncEnumerator(); Assert.IsTrue(await results.MoveNextAsync()); - results.Current.EnableResultSetBuffering(); + results.Current.EnableResultBuffering(); var rows = results.Current.GetAsyncEnumerator(); Assert.IsTrue(await rows.MoveNextAsync()); var borrowed = rows.Current.BorrowFieldMemory(1); diff --git a/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs b/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs index ca21bd4..88e4fd9 100644 --- a/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs +++ b/Slon.Tests/Pg/PublicCommandFlowSurfaceTests.cs @@ -1,4 +1,6 @@ using System.Collections; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; using Slon.Pg; using Slon.Pg.Protocol.Flows; @@ -25,4 +27,25 @@ public void ReplacementIsTheSealedPublicCommandFlowSurface() typeof(bool), typeof(Slon.Pg.Protocol.Flows.CommandFlowOptions).MakeByRefType() ])); } + + [TestMethod] + public void CollectionIsOnTheExperimentalCommandResultSurface() + { + var result = typeof(CommandResult); + var diagnosticId = result.GetCustomAttribute()!.DiagnosticId; + var collect = result.GetMethods().Single(method => + method.Name == nameof(CommandResult.CollectAsync)); + var rowView = result.GetNestedType( + nameof(CommandResult.RowView), BindingFlags.Public)!; + + Assert.IsTrue(collect.IsPublic); + Assert.IsTrue(collect.IsGenericMethodDefinition); + Assert.AreEqual(diagnosticId, + collect.GetCustomAttribute()!.DiagnosticId); + Assert.IsTrue(rowView.IsNestedPublic); + Assert.IsNotNull(rowView.GetMethod( + nameof(CommandResult.RowView.BorrowFieldMemory), BindingFlags.Public | BindingFlags.Instance)); + Assert.AreEqual(diagnosticId, + rowView.GetCustomAttribute()!.DiagnosticId); + } } diff --git a/Slon/Pg/CommandResult.cs b/Slon/Pg/CommandResult.cs index fb58784..dd34669 100644 --- a/Slon/Pg/CommandResult.cs +++ b/Slon/Pg/CommandResult.cs @@ -104,18 +104,18 @@ public RowEnumerator GetAsyncEnumerator(RowBuffering buffering, CancellationToke } /// - /// Retains the memory backing this result set while its rows are enumerated. + /// Retains backend-message memory for this command result until it is released. /// /// - /// Retention may cause subsequent rows to be buffered. Memory returned by + /// Retention may cause subsequent messages to be buffered. Memory returned by /// remains valid until this command result is released. /// - public void EnableResultSetBuffering() + public void EnableResultBuffering() { if (_firstRowEnumerated) ThrowHelper.ThrowInvalidOperation( - "Result-set buffering must be enabled before row enumeration begins."); - _messageEnumerator.EnableResultSetBuffering(); + "Result buffering must be enabled before row enumeration begins."); + _messageEnumerator.EnableResultBuffering(); } public bool TryGetCommandComplete([NotNullWhen(true)]out CommandCompleteMessage? value) @@ -268,16 +268,34 @@ async ValueTask CompleteAsyncCore() EnsureComplete(); } - internal readonly struct RowView + /// A fully buffered PostgreSQL data row supplied to a collection callback. + /// + /// The view normally remains valid only for the duration of its callback. Calling + /// before retains + /// collected views until this command result is released. + /// + [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] + public readonly struct RowView { readonly ReadOnlyMemory _memory; internal RowView(ReadOnlyMemory memory) => _memory = memory; + /// Decodes the field at as . public T GetValue(int ordinal) => BootstrapFieldDecoder.Read(GetFieldSpan(ordinal)); + /// Borrows the field's raw PostgreSQL representation as contiguous memory. + /// + /// The memory normally remains valid only for the duration of the collection callback. + /// Calling before collection retains it until this + /// command result is released. + /// + public ReadOnlyMemory BorrowFieldMemory(int ordinal) + => GetFieldMemory(ordinal); + + /// Decodes the Int32 field at . public int GetInt32(int ordinal) { if (ordinal == 0) @@ -292,30 +310,32 @@ public int GetInt32(int ordinal) return BinaryPrimitives.ReadInt32BigEndian(GetFieldSpan(ordinal)); } - ReadOnlySpan GetFieldSpan(int ordinal) + ReadOnlySpan GetFieldSpan(int ordinal) => GetFieldMemory(ordinal).Span; + + ReadOnlyMemory GetFieldMemory(int ordinal) { ArgumentOutOfRangeException.ThrowIfNegative(ordinal); var fields = _memory.Span; if (fields.Length >= sizeof(short)) { - var remaining = fields[sizeof(short)..]; + var offset = sizeof(short); for (var index = 0; ; index++) { - if (remaining.Length < sizeof(int)) + if (fields.Length - offset < sizeof(int)) ThrowHelper.ThrowInvalidOperation("The DataRow field length is truncated."); - var length = BinaryPrimitives.ReadInt32BigEndian(remaining); - remaining = remaining[sizeof(int)..]; + var length = BinaryPrimitives.ReadInt32BigEndian(fields[offset..]); + offset += sizeof(int); if (length < 0) { if (index == ordinal) ThrowHelper.ThrowInvalidOperation("The requested field is null."); continue; } - if ((uint)length > (uint)remaining.Length) + if ((uint)length > (uint)(fields.Length - offset)) ThrowHelper.ThrowInvalidOperation("The DataRow field is truncated."); if (index == ordinal) - return remaining[..length]; - remaining = remaining[length..]; + return _memory.Slice(offset, length); + offset += length; } } @@ -324,7 +344,18 @@ ReadOnlySpan GetFieldSpan(int ordinal) } } - internal async ValueTask CollectRowsAsync( + /// Collects every row through a synchronous callback. + /// + /// Each complete DataRow body is buffered before is invoked. If the + /// collector throws, the result is drained before that exception is rethrown. Call + /// first when collected values must + /// remain usable after their callback returns. + /// + /// State passed to every collector invocation. + /// The synchronous callback invoked once for each row. + /// A token for cancelling collection. + [Experimental(ExperimentalDiagnostics.PostgreSqlLowerLayer)] + public async ValueTask CollectAsync( TState state, Action collector, CancellationToken cancellationToken = default) { diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs index 9d6f2c5..3f6fdfa 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.MessageEnumerator.cs @@ -73,8 +73,8 @@ public void Initialize(in Command command, PgDecoder decoder) public void Reset() => _messageEnumerator.Reset(); - public void EnableResultSetBuffering() - => _messageEnumerator.EnableResultSetBuffering(); + public void EnableResultBuffering() + => _messageEnumerator.EnableResultBuffering(); public (PgError Error, TransactionStatus TransactionStatus)? CompleteError => _messageEnumerator.CompleteError; @@ -460,7 +460,7 @@ async ValueTask DrainRowsAndComplete(PgDecoder decoder) public void Initialize(in Command command, PgDecoder decoder) { if (_decoder is not null) - _decoder.ResultSetBuffering = false; + _decoder.ResultBuffering = false; _describeOnly = command.DescribeOnly; _withSync = command.WithSync; if (!ReferenceEquals(_decoder, decoder)) @@ -480,7 +480,7 @@ public void Initialize(in Command command, PgDecoder decoder) public void Reset() { if (_decoder is not null) - _decoder.ResultSetBuffering = false; + _decoder.ResultBuffering = false; _describeOnly = false; _withSync = false; _decoder = null!; @@ -493,12 +493,12 @@ public void Reset() _done = true; } - public void EnableResultSetBuffering() + public void EnableResultBuffering() { if (_disposed) ThrowHelper.ThrowInvalidOperation( "The command result has already been released."); - _decoder.ResultSetBuffering = true; + _decoder.ResultBuffering = true; } public (PgError Error, TransactionStatus TransactionStatus)? CompleteError diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index cbcfdf5..433176a 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1881,7 +1881,7 @@ internal void AssignCancellationBoundary(PgClientFlow flow, int window) internal void OnReleasing(PgClientFlow flow) { - Decoder.EndResultSetBuffering(flow); + Decoder.EndResultBuffering(flow); protocol._serverParameterState.CommitFlow(); ClearCancellationActivation(flow); var idle = ActivatedFlow is null; diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index a3855e6..d67ab70 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -32,7 +32,7 @@ public sealed class PgDecoder: IEnumerator, IAsyncEnumerator _pipe.CompleteCurrentMessage(); - internal bool ResultSetBuffering + internal bool ResultBuffering { - get => _resultSetBufferingOwner is not null; + get => _resultBufferingOwner is not null; set { if (value) { var owner = CurrentExecutionControl.Flow; - if (!ReferenceEquals(_resultSetBufferingOwner, owner)) - _resultSetBufferingOwner = owner; - _pipe.EnableResultSetRetention(); + if (!ReferenceEquals(_resultBufferingOwner, owner)) + _resultBufferingOwner = owner; + _pipe.EnableResultRetention(); } else { - if (_resultSetBufferingOwner is null) + if (_resultBufferingOwner is null) return; - _resultSetBufferingOwner = null; - _pipe.EndResultSetRetention(); + _resultBufferingOwner = null; + _pipe.EndResultRetention(); } } } - internal void EndResultSetBuffering(PgClientFlow owner) + internal void EndResultBuffering(PgClientFlow owner) { - if (!ReferenceEquals(_resultSetBufferingOwner, owner)) + if (!ReferenceEquals(_resultBufferingOwner, owner)) return; - _resultSetBufferingOwner = null; - _pipe.EndResultSetRetention(); + _resultBufferingOwner = null; + _pipe.EndResultRetention(); } - void ValidateResultSetBufferingOwner() + void ValidateResultBufferingOwner() { - var owner = _resultSetBufferingOwner; + var owner = _resultBufferingOwner; if (owner is not null && !ReferenceEquals(CurrentExecutionControl.Flow, owner)) - EndResultSetBuffering(owner); + EndResultBuffering(owner); } void PrepareRead() { - ValidateResultSetBufferingOwner(); + ValidateResultBufferingOwner(); _pipe.PrepareRead(); } bool CompleteRead( in ReadResult result, CancellationToken cancellationToken, out bool completed) { - ValidateResultSetBufferingOwner(); + ValidateResultBufferingOwner(); return _pipe.CompleteRead( result, cancellationToken, out completed); } bool ReadNext(TimeSpan timeout) { - ValidateResultSetBufferingOwner(); + ValidateResultBufferingOwner(); return _pipe.MoveNext(timeout); } diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index c12dd32..bc7f703 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -22,7 +22,7 @@ enum PendingRead : byte { None, Messages, Slide, Extend } int _minimumReadSize; PendingRead _pendingRead; bool _hasActiveRead; - bool _retainsResultSet; + bool _retainsResult; public PipeReader PipeReader => reader; public BackendMessage Current => _messageContext.Current; @@ -50,7 +50,7 @@ public void PrepareRead() ThrowHelper.ThrowInvalidOperation( "The current message still has a pending read."); - var retainsResultSet = _retainsResultSet; + var retainsResult = _retainsResult; if (!_hasActiveRead) { @@ -62,7 +62,7 @@ public void PrepareRead() if (_currentMessageLength > 0) { - PrepareAfterPartialMessage(retainsResultSet); + PrepareAfterPartialMessage(retainsResult); return; } @@ -73,13 +73,13 @@ public void PrepareRead() var unreadOffset = checked(_pendingCursorOffset + cursorConsumedLength); var unread = _activeBuffer.GetPosition(unreadOffset); - _pendingCursorOffset = retainsResultSet + _pendingCursorOffset = retainsResult ? unreadOffset : 0; _messageContext.RetireCursor( - retainProjections: retainsResultSet); + retainProjections: retainsResult); reader.AdvanceTo( - retainsResultSet ? _retainedStart : unread, _examined); + retainsResult ? _retainedStart : unread, _examined); _hasActiveRead = false; _activeBuffer = default; _currentMessageLength = -1; @@ -88,12 +88,12 @@ public void PrepareRead() _pendingRead = PendingRead.Messages; } - void PrepareAfterPartialMessage(bool retainsResultSet) + void PrepareAfterPartialMessage(bool retainsResult) { var current = _activeBuffer.Slice(_currentMessageOffset); _messageContext.RetireCursor( - retainProjections: retainsResultSet); - if (retainsResultSet) + retainProjections: retainsResult); + if (retainsResult) { _pendingCursorOffset = checked( _currentMessageOffset + _currentMessageLength); @@ -186,18 +186,18 @@ public bool CompleteRead( if (cursorBuffer.IsEmpty) { completed = result.IsCompleted; - if (completed && !_retainsResultSet) + if (completed && !_retainsResult) _messageContext.RetireCursor(); if (!completed) { reader.AdvanceTo( - _retainsResultSet ? _retainedStart : result.Buffer.End, + _retainsResult ? _retainedStart : result.Buffer.End, result.Buffer.End); _hasActiveRead = false; _activeBuffer = default; - if (!_retainsResultSet) + if (!_retainsResult) _pendingCursorOffset = 0; - _minimumReadSize = _retainsResultSet + _minimumReadSize = _retainsResult ? int.CreateSaturating( _pendingCursorOffset + BackendHeader.ByteCount) : BackendHeader.ByteCount; @@ -300,13 +300,13 @@ void PrepareCurrentMessageRead( throw new ArgumentOutOfRangeException(nameof(consumedLength)); reader.AdvanceTo( - mode is PendingRead.Slide && !_retainsResultSet + mode is PendingRead.Slide && !_retainsResult ? consumed : _retainedStart, _examined); if (mode is PendingRead.Slide) { - if (_retainsResultSet) + if (_retainsResult) { _currentMessageOffset = _activeBuffer.Slice(0, consumed).Length; } @@ -391,18 +391,18 @@ public void SetCurrentMessageLength(long messageLength) public void CompleteCurrentMessage() => _currentMessageLength = -1; - public void EnableResultSetRetention() + public void EnableResultRetention() { if (!_hasActiveRead || _pendingRead is not PendingRead.None) ThrowHelper.ThrowInvalidOperation( - "Result-set retention requires an active backend message."); + "Result retention requires an active backend message."); _ = _messageContext.Current; - _retainsResultSet = true; + _retainsResult = true; } - public void EndResultSetRetention() + public void EndResultRetention() { - _retainsResultSet = false; + _retainsResult = false; if (!_hasActiveRead || _pendingRead is not PendingRead.None || _currentMessageLength > 0 || !_messageContext.TryGetCursorUnread(out var unread)) diff --git a/Slon/Pg/Row.cs b/Slon/Pg/Row.cs index 57e45d9..f7f85cd 100644 --- a/Slon/Pg/Row.cs +++ b/Slon/Pg/Row.cs @@ -69,7 +69,7 @@ public T GetValue(int ordinal) /// that has been reused for unrelated data rather than throw. /// /// - /// Calling before row enumeration extends + /// Calling before row enumeration extends /// the borrow until that command result is released. Copy the memory when it must outlive the /// applicable boundary. /// From 2962a53ba3a532921a7f50d79e71384abb186689 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 17:51:18 +0200 Subject: [PATCH 109/136] Preserve buffered successors after result retention --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 37 +++++++++++++++++++ Slon/Pg/Protocol/BackendMessageContext.cs | 11 ------ Slon/Pg/Protocol/ProtocolReadPipe.cs | 19 +++------- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index f29149a..6140e84 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -460,6 +460,43 @@ public async Task Eof_InvalidatesPublishedBackendMessage() await readPipe.DisposeAsync(); } + [TestMethod] + public async Task EndingResultRetention_ReexposesBufferedSuccessorMessages() + { + var first = BackendMessageBytes(BackendType.DataRow, [0, 0]); + var terminal = BackendMessageBytes(BackendType.CommandComplete, "SELECT 1\0"u8); + var ready = BackendMessageBytes(BackendType.ReadyForQuery, [(byte)'I']); + var successor = BackendMessageBytes(BackendType.BindComplete, []); + var wire = new byte[first.Length + terminal.Length + ready.Length + successor.Length]; + var offset = 0; + foreach (var message in (byte[][])[first, terminal, ready, successor]) + { + message.CopyTo(wire, offset); + offset += message.Length; + } + + var pipe = new Pipe(); + var readPipe = new ProtocolReadPipe(pipe.Reader, + BackendMessageCursor.DefaultDataRowStreamingThreshold); + await pipe.Writer.WriteAsync(wire); + + Assert.IsTrue(await readPipe.MoveNextAsync(default)); + Assert.IsTrue(readPipe.TryMoveNext()); + Assert.AreEqual(BackendType.DataRow, readPipe.Current.Header.Type); + readPipe.EnableResultRetention(); + Assert.IsTrue(readPipe.TryMoveNext()); + Assert.AreEqual(BackendType.CommandComplete, readPipe.Current.Header.Type); + Assert.IsTrue(readPipe.TryMoveNext()); + Assert.AreEqual(BackendType.ReadyForQuery, readPipe.Current.Header.Type); + + readPipe.EndResultRetention(); + Assert.IsTrue(readPipe.TryMoveNext()); + Assert.AreEqual(BackendType.BindComplete, readPipe.Current.Header.Type); + + await pipe.Writer.CompleteAsync(); + await readPipe.DisposeAsync(); + } + [TestMethod] public async Task BackendBodyReader_ExtendsPrefixThenSlides() { diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index ac6fb43..7c621b3 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -519,17 +519,6 @@ public bool TryGetReadRequirement( return true; } - public bool TryGetCursorUnread(out SequencePosition unread) - { - if (!_hasCursor) - { - unread = default; - return false; - } - unread = _cursor.UnreadStart; - return true; - } - // Reads the next message WITHOUT publishing it as Current. The message cursor // really advances past the header, but the parsed (header, buffer) lands in the peek // slot and the follow-up TryMoveNext picks it up without re-parsing. The returned diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index bc7f703..f34c316 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -403,20 +403,11 @@ public void EnableResultRetention() public void EndResultRetention() { _retainsResult = false; - if (!_hasActiveRead || _pendingRead is not PendingRead.None - || _currentMessageLength > 0 - || !_messageContext.TryGetCursorUnread(out var unread)) - { - _messageContext.ReleaseContiguousProjections(); - return; - } - - _messageContext.RetireCursor(); - reader.AdvanceTo(unread, _examined); - _hasActiveRead = false; - _activeBuffer = default; - _currentMessageOffset = 0; - _pendingCursorOffset = 0; + // Keep the active cursor until ordinary message advancement exhausts it. Besides avoiding + // an examined-position rewind (not supported by every PipeReader), this lets a successor + // consume messages already present in the current grant. PrepareRead advances the retained + // prefix when it eventually needs another grant, matching the former batch reader. + _messageContext.ReleaseContiguousProjections(); } public void Dispose() From 10c1840f75387ffb37b8029b30772cb6f5b50afa Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 00:05:10 +0200 Subject: [PATCH 110/136] Trim duplicate command flow async state --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 30 ++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 1cbb7a3..d523720 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -617,7 +617,7 @@ async ValueTask FirstAsync(CancellationToken cancellationToken) Debug.Assert(!_state.ConsumerDetached); RegisterCancellation(cancellationToken); var result = IsSinglePublishedCommand - ? await ReadResultAsync(0).ConfigureAwait(false) + ? await ReadResultAsync().ConfigureAwait(false) : await ReadNextPublishedResultAsync().ConfigureAwait(false); if (result is null) return false; @@ -666,20 +666,22 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) { RegisterCancellation(cancellationToken); var result = _state.Current!; - var resultEnumerator = _state.Context.GetProtocolStatic() - .ResultMessageEnumerator; - await resultEnumerator.DisposeAsync().ConfigureAwait(false); - var completeError = resultEnumerator.CompleteError; + await _state.Context.GetProtocolStatic() + .ResultMessageEnumerator.DisposeAsync().ConfigureAwait(false); + var skipDiscarded = _state.Context.GetProtocolStatic() + .ResultMessageEnumerator.CompleteError + is { TransactionStatus: TransactionStatus.Unknown }; _state.CurrentPublished = false; - if (Volatile.Read(ref _state.ColdState)?.TerminalException is { } consumerFault) + if (Volatile.Read(ref _state.ColdState)?.TerminalException is not null) { Interlocked.Exchange(ref _state.Phase, PhaseDraining); NotifyDrainStarted(); _state.ConsumerDetached = true; await DrainAsync().ConfigureAwait(false); - ExceptionDispatchInfo.Throw(consumerFault); + ExceptionDispatchInfo.Throw( + Volatile.Read(ref _state.ColdState)!.TerminalException!); } - if (completeError is { TransactionStatus: TransactionStatus.Unknown }) + if (skipDiscarded) await SkipDiscardedCommandsAsync().ConfigureAwait(false); _state.CommandIndex++; @@ -732,7 +734,7 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) CommandResult? result = _state.Current; while (_state.CommandIndex < _state.Commands.Count) { - result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); + result = await ReadResultAsync().ConfigureAwait(false); _state.Current = result; _state.CurrentPublished = false; if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) @@ -812,7 +814,7 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) } // Reads through the command's execute prelude and initializes the protocol-static result. - async ValueTask ReadResultAsync(int commandIndex) + async ValueTask ReadResultAsync() { var context = _state.Context; var decoder = context.Decoder; @@ -821,7 +823,7 @@ async ValueTask ReadResultAsync(int commandIndex) throw context.FlowTerminationException; PgError? error; RowDescription? requestedRowDescription; - ref readonly var command = ref _state.Commands.ItemRef(commandIndex); + ref readonly var command = ref _state.Commands.ItemRef(_state.CommandIndex); var describeOnly = command.DescribeOnly; var hasPreparedDescription = command.Descriptor is { IsPrepared: true, PreparedRowDescription: not null }; @@ -870,7 +872,7 @@ async ValueTask ReadResultAsync(int commandIndex) .ConfigureAwait(false); } return InitializeResult( - commandIndex, error, requestedRowDescription, preparationParameterTypes); + _state.CommandIndex, error, requestedRowDescription, preparationParameterTypes); } CommandResult ReadResult(int commandIndex) @@ -1063,7 +1065,7 @@ async ValueTask DrainAsync() await CompleteBatchAsync().ConfigureAwait(false); return; } - result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); + result = await ReadResultAsync().ConfigureAwait(false); } while (true) @@ -1075,7 +1077,7 @@ async ValueTask DrainAsync() await SkipDiscardedCommandsAsync().ConfigureAwait(false); if (++_state.CommandIndex >= _state.Commands.Count) break; - result = await ReadResultAsync(_state.CommandIndex).ConfigureAwait(false); + result = await ReadResultAsync().ConfigureAwait(false); } await CompleteBatchAsync().ConfigureAwait(false); } From 2be3748776538ad0ab03838fe341bf6280a4a809 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 00:07:39 +0200 Subject: [PATCH 111/136] Reuse protocol result state across completion awaits --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index d523720..7378d4b 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -929,9 +929,10 @@ CommandResult InitializeResult( async ValueTask<(PgError Error, TransactionStatus TransactionStatus)?> CompleteCurrentResultAsync() { - var enumerator = _state.Context.GetProtocolStatic().ResultMessageEnumerator; - await enumerator.DisposeAsync().ConfigureAwait(false); - return enumerator.CompleteError; + await _state.Context.GetProtocolStatic() + .ResultMessageEnumerator.DisposeAsync().ConfigureAwait(false); + return _state.Context.GetProtocolStatic() + .ResultMessageEnumerator.CompleteError; } (PgError Error, TransactionStatus TransactionStatus)? CompleteCurrentResult() From b6faab7269f461b160ea5c55e5afb55748d13499 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 00:19:38 +0200 Subject: [PATCH 112/136] Split command result read state machines --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 125 +++++++++++++------------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 7378d4b..8f2bb8a 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -255,6 +255,8 @@ readonly struct CommandFlowCore(TOps ops) readonly TOps _ops = ops; ref CommandExecutionState _state => ref _ops.GetField(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void SetCurrent(CommandResult result) => _state.Current = result; internal bool IsResultReady => Volatile.Read(ref _state.Phase) is PhaseResultReady; bool IsSinglePublishedCommand => _state.Commands.Count is 1 @@ -731,17 +733,15 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask ReadNextPublishedResultAsync() { - CommandResult? result = _state.Current; while (_state.CommandIndex < _state.Commands.Count) { - result = await ReadResultAsync().ConfigureAwait(false); - _state.Current = result; + SetCurrent(await ReadResultAsync().ConfigureAwait(false)); _state.CurrentPublished = false; if (!_state.Commands.ItemRef(_state.CommandIndex).SuppressEnumeration) - return result; + return _state.Current; var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); - var suppressedError = result.Error; + var suppressedError = _state.Current!.Error; if (suppressedError is null && completeError is { } completionError) suppressedError = completionError.Error; if (suppressedError is not null) @@ -764,7 +764,7 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) _state.CommandIndex++; } - if (result is null) + if (_state.Current is null) throw ThrowHelper.ThrowInvalidOperation("The flow contains no commands."); await CompleteBatchAsync().ConfigureAwait(false); _state.ConsumerObservedCompletion = true; @@ -813,66 +813,72 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) return null; } - // Reads through the command's execute prelude and initializes the protocol-static result. - async ValueTask ReadResultAsync() + // Dispatch before entering an async machine so each mutually-exclusive protocol shape carries + // only its own awaiter and scratch state. + ValueTask ReadResultAsync() { - var context = _state.Context; - var decoder = context.Decoder; // After close, a fresh command must not consume bytes left by its predecessor. - if (context.IsProtocolClosed) - throw context.FlowTerminationException; - PgError? error; - RowDescription? requestedRowDescription; + if (_state.Context.IsProtocolClosed) + return ValueTask.FromException(_state.Context.FlowTerminationException); ref readonly var command = ref _state.Commands.ItemRef(_state.CommandIndex); - var describeOnly = command.DescribeOnly; - var hasPreparedDescription = command.Descriptor - is { IsPrepared: true, PreparedRowDescription: not null }; - decoder.UseReadTimeout(command.Timeout); - ParameterTypeList? preparationParameterTypes = null; + _state.Context.Decoder.UseReadTimeout(command.Timeout); if (command.DescribeForPreparation) + return ReadPreparationResultAsync(); + return command.Descriptor is { IsPrepared: true, PreparedRowDescription: not null } + && !command.DescribeOnly + ? ReadPreparedResultAsync() + : ReadUnpreparedResultAsync(); + } + + async ValueTask ReadPreparationResultAsync() + { + ref readonly var command = ref _state.Commands.ItemRef(_state.CommandIndex); + var preparation = await command.ReadPreparationDescriptionAsync( + _state.Context.Decoder, + _state.Context.GetProtocolStatic().RowDescription) + .ConfigureAwait(false); + return InitializeResult( + _state.CommandIndex, preparation.Item1, preparation.Item3, preparation.Item2); + } + + async ValueTask ReadPreparedResultAsync() + { + // Prepared commands with a known description have the compact BindComplete -> + // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes this + // frame rather than a nested parser coroutine. + if (!_state.Context.Decoder.TryMoveNext()) { - var preparation = await command.ReadPreparationDescriptionAsync( - decoder, context.GetProtocolStatic().RowDescription) - .ConfigureAwait(false); - error = preparation.Item1; - preparationParameterTypes = preparation.Item2; - requestedRowDescription = preparation.Item3; + if (!await _state.Context.Decoder.MoveNextAsync().ConfigureAwait(false)) + _state.Context.Decoder.ThrowUnexpectedEof(); } - else if (hasPreparedDescription && !describeOnly) + var message = _state.Context.Decoder.Current; + PgError? error; + if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) { - // Prepared commands with a known description have the compact BindComplete -> - // DataRow/CommandComplete prelude. Await the decoder directly so a read wake resumes this - // frame rather than a nested parser coroutine. - if (!decoder.TryMoveNext()) - { - if (!await decoder.MoveNextAsync().ConfigureAwait(false)) - decoder.ThrowUnexpectedEof(); - } - var message = decoder.Current; - if (message.EnsureExpectedOrError(PgTypes.BackendType.BindComplete) is { } bindError) - { - error = bindError; - } - else - { - if (!decoder.TryMoveNext()) - { - if (!await decoder.MoveNextAsync().ConfigureAwait(false)) - decoder.ThrowUnexpectedEof(); - } - decoder.Current.DebugEnsureExpected(PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); - error = null; - } - requestedRowDescription = null; + error = bindError; } else { - (error, requestedRowDescription) = await command - .ReadUntilExecuteAsync(decoder, context.GetProtocolStatic().RowDescription) - .ConfigureAwait(false); + if (!_state.Context.Decoder.TryMoveNext()) + { + if (!await _state.Context.Decoder.MoveNextAsync().ConfigureAwait(false)) + _state.Context.Decoder.ThrowUnexpectedEof(); + } + _state.Context.Decoder.Current.DebugEnsureExpected( + PgTypes.BackendType.DataRow, PgTypes.BackendType.CommandComplete); + error = null; } - return InitializeResult( - _state.CommandIndex, error, requestedRowDescription, preparationParameterTypes); + return InitializeResult(_state.CommandIndex, error, null); + } + + async ValueTask ReadUnpreparedResultAsync() + { + ref readonly var command = ref _state.Commands.ItemRef(_state.CommandIndex); + var result = await command.ReadUntilExecuteAsync( + _state.Context.Decoder, + _state.Context.GetProtocolStatic().RowDescription) + .ConfigureAwait(false); + return InitializeResult(_state.CommandIndex, result.Item1, result.Item2); } CommandResult ReadResult(int commandIndex) @@ -1055,8 +1061,7 @@ async ValueTask DrainAsync() { try { - var result = _state.Current; - if (result is null) + if (_state.Current is null) { await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); if (_state.CommandIndex < 0) @@ -1066,19 +1071,19 @@ async ValueTask DrainAsync() await CompleteBatchAsync().ConfigureAwait(false); return; } - result = await ReadResultAsync().ConfigureAwait(false); + SetCurrent(await ReadResultAsync().ConfigureAwait(false)); } while (true) { var completeError = await CompleteCurrentResultAsync().ConfigureAwait(false); - CaptureDrainError(result, completeError); + CaptureDrainError(_state.Current!, completeError); _state.CurrentPublished = false; if (completeError is { TransactionStatus: TransactionStatus.Unknown }) await SkipDiscardedCommandsAsync().ConfigureAwait(false); if (++_state.CommandIndex >= _state.Commands.Count) break; - result = await ReadResultAsync().ConfigureAwait(false); + SetCurrent(await ReadResultAsync().ConfigureAwait(false)); } await CompleteBatchAsync().ConfigureAwait(false); } From c35f529b9b535fbb2476f962bc0965e45cef15e9 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 01:16:30 +0200 Subject: [PATCH 113/136] Avoid generic source driver dispatch wrapper --- Slon/Pg/Protocol/PgFlowSourceDriver.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Slon/Pg/Protocol/PgFlowSourceDriver.cs b/Slon/Pg/Protocol/PgFlowSourceDriver.cs index 1a72561..55f7799 100644 --- a/Slon/Pg/Protocol/PgFlowSourceDriver.cs +++ b/Slon/Pg/Protocol/PgFlowSourceDriver.cs @@ -8,6 +8,8 @@ namespace Slon.Pg.Protocol; /// sealed class PgFlowSourceDriver { + static readonly Action RunAction = static state => ((PgFlowSourceDriver)state!).Run(); + readonly PgClientFlowSource.State _source; readonly SourceWakeEvent _wakeEvent; readonly Action _signalHeldSyncFlow; @@ -53,7 +55,7 @@ public void Drive(bool runContinuationsAsynchronously) return; if (runContinuationsAsynchronously) - _wakeEvent.Scheduler.SubmitDetached(static driver => driver.Run(), this); + _wakeEvent.Scheduler.SubmitDetached(RunAction, (object?)this); else Run(); } @@ -114,7 +116,7 @@ void OnWaitReady(SourceWakeEvent.WaitReadyContext context) } void ScheduleRun() - => _wakeEvent.Scheduler.SubmitDetached(static driver => driver.Run(), this); + => _wakeEvent.Scheduler.SubmitDetached(RunAction, (object?)this); void Run() { @@ -149,7 +151,7 @@ void Run() } if (transfer) - _wakeEvent.Scheduler.SubmitDetached(static driver => driver.Run(), this); + _wakeEvent.Scheduler.SubmitDetached(RunAction, (object?)this); return; } } From 9a95cad13d66bf99342b224b423d9815bb86115b Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 08:36:39 +0200 Subject: [PATCH 114/136] Use runtime async for normal pipe operations on net11 --- Slon/Pipelines/StreamPipeReader.cs | 8 ++++++++ Slon/Pipelines/StreamPipeWriter.cs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/Slon/Pipelines/StreamPipeReader.cs b/Slon/Pipelines/StreamPipeReader.cs index 418e580..a311e0a 100644 --- a/Slon/Pipelines/StreamPipeReader.cs +++ b/Slon/Pipelines/StreamPipeReader.cs @@ -12,7 +12,9 @@ interface IStreamOwner : IDisposable, IAsyncDisposable { } abstract class StreamPipeReader : PipeReader { +#if !NET11_0_OR_GREATER readonly ValueTaskSourcePromise _readAsyncCorePromise = new(); +#endif bool _directReadAwaitingData; readonly IStreamOwner? _streamOwner; int _isReadActive; @@ -379,18 +381,24 @@ protected ReadResult ReadCore(int minimumSize, TimeSpan timeout) protected ValueTask ReadAsyncCore(int minimumSize, CancellationToken cancellationToken) { +#if !NET11_0_OR_GREATER PromiseAsyncValueTaskMethodBuilder.Promise = _readAsyncCorePromise; try { +#endif return ReadAsyncCore(minimumSize, PendingReadTokenSource, cancellationToken); +#if !NET11_0_OR_GREATER } finally { PromiseAsyncValueTaskMethodBuilder.Promise = null; } +#endif +#if !NET11_0_OR_GREATER [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] +#endif async ValueTask ReadAsyncCore(int minimumSize, AutoResetCancellationTokenSource? tokenSource, CancellationToken cancellationToken) { diff --git a/Slon/Pipelines/StreamPipeWriter.cs b/Slon/Pipelines/StreamPipeWriter.cs index 417561c..4bb0a87 100644 --- a/Slon/Pipelines/StreamPipeWriter.cs +++ b/Slon/Pipelines/StreamPipeWriter.cs @@ -9,7 +9,9 @@ namespace Slon.Pipelines; abstract class StreamPipeWriter : PipeWriter, IOutputWriter { +#if !NET11_0_OR_GREATER readonly ValueTaskSourcePromise _flushAsyncCorePromise = new(); +#endif readonly IStreamOwner? _streamOwner; bool _isFlushActive; @@ -308,18 +310,24 @@ protected virtual FlushResult FlushCore(bool writeToStream, ReadOnlySpan d protected virtual ValueTask FlushAsyncCore(bool writeToStream, ReadOnlyMemory data, CancellationToken cancellationToken) { +#if !NET11_0_OR_GREATER PromiseAsyncValueTaskMethodBuilder.Promise = _flushAsyncCorePromise; try { +#endif return FlushAsyncCore(PendingFlushTokenSource, writeToStream, data, cancellationToken); +#if !NET11_0_OR_GREATER } finally { PromiseAsyncValueTaskMethodBuilder.Promise = null; } +#endif +#if !NET11_0_OR_GREATER [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] +#endif async ValueTask FlushAsyncCore(AutoResetCancellationTokenSource? tokenSource, bool writeToStream, ReadOnlyMemory data, CancellationToken cancellationToken) { // Cancellation token was already checked before getting here. From 7a4eb3644088ba9545528a77422d7af03ae47df9 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 09:19:13 +0200 Subject: [PATCH 115/136] Upgrade initial publication timestamp proof --- provenance/initial-publication.tag.ots | Bin 527 -> 3742 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/provenance/initial-publication.tag.ots b/provenance/initial-publication.tag.ots index ab2ccbfe317c6d6a85f4584c10dd8b512c3f1d89..6957b0d6e539fcbf97da02f0580bd59e8824159d 100644 GIT binary patch delta 3269 zcmY+Gc{G&m8^>o*Mudc6q%4swyeY}PWvP%Q`xYTv8B3Tj_BAH!L|G=g7}-KX5+-C2 z$-W!1?@NZ?^LXF)ocH(7bIyJ4&-s3@=Q{WGJr{;5gY#tr^ff_X>W4nrU%%amZk( z)N0?<`jl2&DV5#j{MS16(oBki9uI)5O|~FuOVIjOexV6@tO$hx$&en}4E_S%WXr9z zv({PN028~=Sz1wrMO2pz^p?K;oC{NwpG!QCNeFVt+M|`(bq9)>V2}+A4R7NLUz~Kp z8e)RKtCujXgf@JiQ*L&V_gF~=5F1Ojh1_SIkH?N~D{JQG$-dyPS4YWrSXu;JAfYv9 z=y(8TM<3m6YfQWwm|tI)T3rg)ikc;A&fYx?sPT;m6N&_jX%=|2mDOjo*>W(Rl4LLL ze#zqH>(qvi^*2v*-Uv(eK>&#IIwl+^SJ3UGH$F-w%{6n4M`s`ZDy{fJrRF$kGh7fr z7M_<=+z_z27UgU#vwj+V1=!lKp_waR6R0H|09eX_&a!L zu2|2vv5e%lKruhG)FeOLm4_UxP# ztHF8UYL6!s2inybE?!iX1$B57d4y}%O+L9Q_hCL&tLcZ8u5QGSTGyRZm91+!xJ(9_ zEb5uxYWe$_bSQLZnb?vmO=-182t1!8PjHYz^->1wfKZO0q^Ozh5al$>!Nsep6xV2e z^%sas&aPE1!AUjOO>cq)C6aVH(t6U;GR?YtC^h83u!#icC4{F~AO`{IM(@zpf5FPLzUP1@q&)&(1m6QE}Ys{J}Rc zT=3;*{Y2BwZ-LtKr(PO3S~}YS2%~sFwA{xZGV&US{K?%G&rKZLqL6IU<9@RtwKCy` z1OPdT)*Q*Tp;K-2afu7)?Uiob;=6~ZoueKP_k@sUVgmwH~WV68Ta7c@P z;hb+0`b<~(!g?;gas`Se7y=Ak$Q+LsqV7uO#&Q77&*wWja!LCuU#R`{eKp8D2lZ?K zVw8X2__oGXO+IMit6Ob^qqLY^TA*S);c0-IehKz83MeMC2~Wz%9N@i=heo?@m0GnB zeQpTDAG=bxX9^t^s$l??ZaXxm@f14wIuG4l?UsB8x<%Uf<%Zi$7ta2e(hZE$=l9_urZv~XE zc$9>(bR_&tuOsBO11j9->pGZ~x!QCrzi^9jpw=aSaf<~Vn#-;PFyg3O zA%00z*!Nqv&f3pr=G(6wI~;t24N2kd5^f(++5t|U+R8C|CIW{n#iwUJ8HK5YAS&+0 zv@6d|)p*2KF(n}Zg#7*m8+2+@;wW~VAe!_RcPNZJLlSOZG?fAeG6l&3V2UUead&5t++pK`vE)*oo# z{{@|_ry=uyYUWs zKZwmzTz^M>$BM^771R8U7nmSmT>CafXxd^nitD{t4&auKe;9qW{cXMMEXaR|i%*2{ zAcOT$k1q6>8Q%;W>t-$%>MwFak}IF&=}2;VpvzB}@O=c$u8fAIr^P$txV*%Va{HLQ zepKrUgst9^rhi%~Wo*Rp8Wb|*n9iCXRpHRIs#4~P^^u0K6s+v!)lM)>h$5u(zMhP9 zq@J_p;qRWE{t3~k#*)v6Z5(uvprnfVC~t0~ts=J-(2IMCE5n%BtH2$IW*eYR6Q!s; zpcO3|_N2%fuxk)n!3Kb4U&v`CtLgGrm4a+0&?%GierNCe8X?OrDWP%{L@sQpQqm-cVtVp@c?Kx#1!fZL6+9}e(n(Lu~NOd$=P*^ z$(6%n>Z>`H#`KHqcMxLiU@^c1(^Lu$&a`4ST|w$W&;RD%ociTjM$RS z`$0)Dm0MP^cL~iGDKQiMi*D;M9G(oWLzuQhDR%bvF$i-H?}_r`hl^N~Ao0w?fq<4f zq0J=Qa{#hmac+uiT|yXs+4!Ngcgp@L{LG%6U7NPRd=1G{Vh;=%1{?2fJO>d2-C*A==%cpli0vvGfkDGUKn9gLvNmGj(8({~ z7Xyqa#%wZRPv9V~%bt4nmjg~IYKQofll4r3h`{~*vyCZ$Q%40d19y|-sN&#(*{OX4 zD1Cwkq#qu^&^G-=cVYw|@^3m|9*F4Ohx%1q+7j*vi^yWKBCo)#3g`M-D|{lT^MLZOjx{JB;zSm^a5fo?NF}agLZf+bF9Fk1`53;`a6o8W`AR1On>It z6*XH1vNyfEtQWukUy>DHR_jtgA+A<;mccSX!$FUk700&Rq}@;Q*6y3z__3;FpHx1f z3!n~vf^ACgO_iAmbU4q$;!O#?uJlkmO2R_n%ad4-AM9Nq!;z~X3hAA^YT-5Hr4UHQ^mi*z z2)7hCmsjr|2~U2_F1Ot+<7dte)2wfzR*6elG=RSj28%8XOZqeY36EPyl<`|V9ziv^ a{ZRaQsQf1+D2rLr_+z#6f8(KhS^9r~iWw0A delta 25 hcmbOy+s`s#-^BgKlNDIqCugwgPJYf=GC79J3;=@73331c From 4d88eaf4f6cadd21b907e946093a5d7be428366d Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 11:09:36 +0200 Subject: [PATCH 116/136] Advance retained results to their active buffer origin --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 83 +++++++++++++++++++ .../Pipelines/SegmentChainBuilderTests.cs | 45 ++++++++++ Slon/Pg/Protocol/BackendMessageContext.cs | 9 ++ Slon/Pg/Protocol/BackendMessageCursor.cs | 24 ++++++ Slon/Pg/Protocol/ProtocolReadPipe.cs | 44 +++++++++- 5 files changed, 201 insertions(+), 4 deletions(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index 6140e84..7fa1622 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -44,10 +44,12 @@ sealed class RejectRetiredSuppliedReadReader(PipeReader inner) : PipeReader { ReadResult _activeRead; bool _rejectAdvanceAtStart; + long? _expectedAdvanceOffset; public Action? BeforeAdvance { get; set; } public void RejectAdvanceAtActiveStart() => _rejectAdvanceAtStart = true; + public void ExpectAdvanceAtActiveOffset(long offset) => _expectedAdvanceOffset = offset; public override ValueTask ReadAsync(CancellationToken cancellationToken = default) { @@ -74,6 +76,12 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami if (_rejectAdvanceAtStart && consumed.Equals(_activeRead.Buffer.Start)) Assert.Fail("The supplied read was retired before its buffer was inspected."); _rejectAdvanceAtStart = false; + if (_expectedAdvanceOffset is { } expectedOffset) + { + Assert.AreEqual(_activeRead.Buffer.GetPosition(expectedOffset), consumed, + "The read pipe retained bytes before the active result tenure."); + _expectedAdvanceOffset = null; + } inner.AdvanceTo(consumed, examined); } @@ -192,6 +200,46 @@ public void BackendMessageCursor_AdvancesAcrossExactSegmentBoundary() Assert.IsFalse(cursor.TryReadNextInPlace(out _, out _, out _)); } + [TestMethod] + public void BackendMessageCursor_CollapsesRemainingFinalSegmentToArrayBacking() + { + var first = BackendMessageBytes(BackendType.CommandComplete, 6); + var second = BackendMessageBytes(BackendType.ReadyForQuery, 6); + var cursor = new BackendMessageCursor(Segmented(first, second)); + + Assert.IsTrue(cursor.TryReadNextInPlace(out _, out _, out _)); + Assert.IsTrue(cursor.TryReadNextInPlace(out _, out var remaining, out _)); + + Assert.IsInstanceOfType(remaining.Start.GetObject()); + Assert.AreSame(second, remaining.Start.GetObject()); + CollectionAssert.AreEqual(second, remaining.ToArray()); + } + + [TestMethod] + public void BackendMessageCursor_CollapsesFinalSegmentAfterStraddlingMessage() + { + var straddling = BackendMessageBytes( + BackendType.DataRow, new byte[] { 1, 2, 3, 4, 5, 6 }); + var remainingMessage = BackendMessageBytes(BackendType.ReadyForQuery, 6); + const int firstSegmentLength = 7; + var finalSegment = new byte[straddling.Length - firstSegmentLength + + remainingMessage.Length + 6]; + straddling.AsSpan(firstSegmentLength).CopyTo(finalSegment.AsSpan(3)); + remainingMessage.CopyTo(finalSegment.AsSpan( + 3 + straddling.Length - firstSegmentLength)); + var finalMemory = finalSegment.AsMemory(3, + straddling.Length - firstSegmentLength + remainingMessage.Length); + var cursor = new BackendMessageCursor(Segmented( + straddling.AsMemory(0, firstSegmentLength), finalMemory)); + + Assert.IsTrue(cursor.TryReadNextInPlace(out _, out var first, out _)); + Assert.IsFalse(first.IsSingleSegment); + Assert.IsTrue(cursor.TryReadNextInPlace(out _, out var remaining, out _)); + + Assert.AreSame(finalSegment, remaining.Start.GetObject()); + CollectionAssert.AreEqual(remainingMessage, remaining.ToArray()); + } + [TestMethod] public void ContiguousMemory_StraddleIsProjectedOncePerResultTenure() { @@ -497,6 +545,41 @@ public async Task EndingResultRetention_ReexposesBufferedSuccessorMessages() await readPipe.DisposeAsync(); } + [TestMethod] + public async Task BeginningNewResultRetention_DropsPriorResultPrefixAtNextRead() + { + var prior = BackendMessageBytes(BackendType.DataRow, [0, 1]); + var retained = BackendMessageBytes(BackendType.DataRow, [2, 3]); + var terminal = BackendMessageBytes(BackendType.CommandComplete, "SELECT 1\0"u8); + var firstGrant = new byte[prior.Length + retained.Length]; + prior.CopyTo(firstGrant, 0); + retained.CopyTo(firstGrant, prior.Length); + + var pipe = new Pipe(); + var reader = new RejectRetiredSuppliedReadReader(pipe.Reader); + var readPipe = new ProtocolReadPipe(reader, + BackendMessageCursor.DefaultDataRowStreamingThreshold); + await pipe.Writer.WriteAsync(firstGrant); + + Assert.IsTrue(await readPipe.MoveNextAsync(default)); + Assert.IsTrue(readPipe.TryMoveNext()); + readPipe.EnableResultRetention(); + readPipe.EndResultRetention(); + Assert.IsTrue(readPipe.TryMoveNext()); + readPipe.EnableResultRetention(); + Assert.IsFalse(readPipe.TryMoveNext()); + + reader.ExpectAdvanceAtActiveOffset(prior.Length); + await pipe.Writer.WriteAsync(terminal); + Assert.IsTrue(await readPipe.MoveNextAsync(default)); + Assert.IsTrue(readPipe.TryMoveNext()); + Assert.AreEqual(BackendType.CommandComplete, readPipe.CurrentType); + + readPipe.EndResultRetention(); + await pipe.Writer.CompleteAsync(); + await readPipe.DisposeAsync(); + } + [TestMethod] public async Task BackendBodyReader_ExtendsPrefixThenSlides() { diff --git a/Slon.Tests/Pipelines/SegmentChainBuilderTests.cs b/Slon.Tests/Pipelines/SegmentChainBuilderTests.cs index c71d4ac..fb9c658 100644 --- a/Slon.Tests/Pipelines/SegmentChainBuilderTests.cs +++ b/Slon.Tests/Pipelines/SegmentChainBuilderTests.cs @@ -6,6 +6,51 @@ namespace Slon.Tests.Pipelines; [TestClass] public class SegmentChainBuilderTests { + const int ReaderBufferSize = 65536; + const int MinimumReadSize = 512; + + [TestMethod] + public void ReaderReserve_WithConsumedPrefix_ConsolidatesRemainingBytes() + { + using var builder = new SegmentChainBuilder( + MemoryPool.Shared, ReaderBufferSize, MinimumReadSize, + retainBufferOnEmpty: true); + var memory = builder.Reserve(ReaderBufferSize, enforceHint: true); + memory.Span[^1] = 42; + builder.Grow(ReaderBufferSize); + var initial = builder.GetReadOnlySequence(); + var initialHead = builder.HeadInfo.Head; + + builder.AdvanceTo(initial.GetPosition(ReaderBufferSize - 1024)); + builder.Reserve(MinimumReadSize, enforceHint: true); + + var consolidated = builder.GetReadOnlySequence(); + Assert.IsTrue(consolidated.IsSingleSegment); + Assert.AreNotSame(initialHead, builder.HeadInfo.Head); + Assert.AreEqual(0, builder.HeadInfo.Index); + Assert.AreEqual(1024, consolidated.Length); + Assert.AreEqual(42, consolidated.FirstSpan[^1]); + } + + [TestMethod] + public void ReaderReserve_AtUnadvancedHead_CreatesSecondSegment() + { + using var builder = new SegmentChainBuilder( + MemoryPool.Shared, ReaderBufferSize, MinimumReadSize, + retainBufferOnEmpty: true); + builder.Reserve(ReaderBufferSize, enforceHint: true).Span[^1] = 42; + builder.Grow(ReaderBufferSize); + var initialHead = builder.HeadInfo.Head; + + builder.Reserve(MinimumReadSize, enforceHint: true); + + var chained = builder.GetReadOnlySequence(); + Assert.IsFalse(chained.IsSingleSegment); + Assert.AreSame(initialHead, builder.HeadInfo.Head); + Assert.AreEqual(ReaderBufferSize, chained.Length); + Assert.AreEqual(42, chained.FirstSpan[^1]); + } + [TestMethod] public void AdvanceToEmpty_RetainsSegmentAndOwnedMemory_WhenEnabled() { diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index 7c621b3..e81a743 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -234,6 +234,15 @@ public long GetCurrentMessageOffset(short token) return _currentMessageOffset; } + internal long CaptureCurrentMessageOffset() + => GetCurrentMessageOffset(_version); + + internal void RebaseCurrentMessageOffset() + { + Debug.Assert((_messageState & MessageOffsetCaptured) != 0); + _currentMessageOffset = 0; + } + public ReadOnlyMemory GetContiguousMemory( short token, ReadOnlyMemory source) { diff --git a/Slon/Pg/Protocol/BackendMessageCursor.cs b/Slon/Pg/Protocol/BackendMessageCursor.cs index d366b8f..665dc35 100644 --- a/Slon/Pg/Protocol/BackendMessageCursor.cs +++ b/Slon/Pg/Protocol/BackendMessageCursor.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using Slon.Pipelines; using Slon.Runtime.CompilerServices; using static Slon.Pg.Protocol.PgTypes; @@ -241,6 +242,7 @@ public FastReadOnlySequence SplitInPlace(long offset) _startObject = next; _startIndex = SegmentFlag; _length -= offset; + NormalizeSingleSegmentArray(); return boundaryPrefix; } if ((ulong)offset < (uint)firstLength) @@ -265,8 +267,30 @@ FastReadOnlySequence SplitSlow(long offset) var remaining = sequence.Slice(offset); var result = new FastReadOnlySequence(prefix); this = new(remaining); + NormalizeSingleSegmentArray(); return result; } + // ReadOnlySequence.Slice retains segment positions after a multi-segment sequence has + // advanced wholly into its final segment. PipeReader-style consumers should not continue + // paying that historical topology: when the remaining segment exposes array memory, carry + // the actual array and absolute indices from this point forward. + [MethodImpl(MethodImplOptions.NoInlining)] + void NormalizeSingleSegmentArray() + { + if (!IsSegmentBacked || !ReferenceEquals(_startObject, _endObject)) + return; + + var memory = ((ReadOnlySequenceSegment)_startObject!).Memory; + if (!MemoryMarshal.TryGetArray(memory, out ArraySegment array)) + return; + + var startIndex = StartIndex; + var endIndex = EndIndex; + _startObject = _endObject = array.Array; + _startIndex = array.Offset + startIndex; + _endIndex = array.Offset + endIndex; + } + } } diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index f34c316..d458136 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -15,6 +15,9 @@ enum PendingRead : byte { None, Messages, Slide, Extend } ReadOnlySequence _activeBuffer; SequencePosition _examined; SequencePosition _retainedStart; + // Offset of _retainedStart within _activeBuffer. Reset to zero whenever a new PipeReader grant + // begins at that position; non-zero only while advancing the retention origin within one grant. + long _retainedOffset; long _currentMessageOffset; long _currentMessageLength = -1; long _pendingCursorOffset; @@ -72,9 +75,10 @@ public void PrepareRead() "The current backend-message cursor has not been exhausted."); var unreadOffset = checked(_pendingCursorOffset + cursorConsumedLength); + var retainedOffset = retainsResult ? _retainedOffset : 0; var unread = _activeBuffer.GetPosition(unreadOffset); _pendingCursorOffset = retainsResult - ? unreadOffset + ? checked(unreadOffset - retainedOffset) : 0; _messageContext.RetireCursor( retainProjections: retainsResult); @@ -82,6 +86,7 @@ public void PrepareRead() retainsResult ? _retainedStart : unread, _examined); _hasActiveRead = false; _activeBuffer = default; + _retainedOffset = 0; _currentMessageLength = -1; _currentMessageOffset = 0; _minimumReadSize = int.CreateSaturating(requiredLength); @@ -96,7 +101,7 @@ void PrepareAfterPartialMessage(bool retainsResult) if (retainsResult) { _pendingCursorOffset = checked( - _currentMessageOffset + _currentMessageLength); + _currentMessageOffset + _currentMessageLength - _retainedOffset); var examined = current.Length >= _currentMessageLength ? current.GetPosition(_currentMessageLength) : _examined; @@ -122,6 +127,7 @@ void PrepareAfterPartialMessage(bool retainsResult) _hasActiveRead = false; _activeBuffer = default; + _retainedOffset = 0; _currentMessageLength = -1; _currentMessageOffset = 0; _pendingRead = PendingRead.Messages; @@ -157,6 +163,7 @@ public bool CompleteRead( _activeBuffer = result.Buffer; _examined = result.Buffer.End; _retainedStart = result.Buffer.Start; + _retainedOffset = 0; _hasActiveRead = true; if (_pendingSkipLength > 0) { @@ -299,6 +306,7 @@ void PrepareCurrentMessageRead( && (consumedLength <= 0 || consumedLength >= _currentMessageLength)) throw new ArgumentOutOfRangeException(nameof(consumedLength)); + var retainedOffset = _retainsResult ? _retainedOffset : 0; reader.AdvanceTo( mode is PendingRead.Slide && !_retainsResult ? consumed @@ -308,7 +316,8 @@ mode is PendingRead.Slide && !_retainsResult { if (_retainsResult) { - _currentMessageOffset = _activeBuffer.Slice(0, consumed).Length; + _currentMessageOffset = checked( + _activeBuffer.Slice(0, consumed).Length - retainedOffset); } else { @@ -317,6 +326,12 @@ mode is PendingRead.Slide && !_retainsResult } _currentMessageLength -= consumedLength; } + else if (_retainsResult) + { + _currentMessageOffset = checked( + _currentMessageOffset - retainedOffset); + } + _retainedOffset = 0; _hasActiveRead = false; _activeBuffer = default; _pendingRead = mode; @@ -396,7 +411,28 @@ public void EnableResultRetention() if (!_hasActiveRead || _pendingRead is not PendingRead.None) ThrowHelper.ThrowInvalidOperation( "Result retention requires an active backend message."); - _ = _messageContext.Current; + if (_retainsResult) + { + _ = _messageContext.Current; + return; + } + // A prior result may have completed entirely within this PipeReader grant. Its borrowed + // values were released by EndResultRetention, so the next tenure can move the retained + // origin to its own first message. Rebase offsets when the eventual read starts there. + var retainedOffset = checked(_pendingCursorOffset + + _messageContext.CaptureCurrentMessageOffset()); + if (_currentMessageLength > 0) + { + // Before the next grant these describe the cursor base and the message end relative to + // that base. Dropping preceding messages makes the retained message its own base, so + // preserve only its actual length and carry its current absolute offset until AdvanceTo. + _currentMessageLength = checked(_currentMessageLength + - (retainedOffset - _currentMessageOffset)); + _currentMessageOffset = retainedOffset; + _messageContext.RebaseCurrentMessageOffset(); + } + _retainedStart = _activeBuffer.GetPosition(retainedOffset); + _retainedOffset = retainedOffset; _retainsResult = true; } From a10f0b12afd1c6aacefe9637f6ae187fd0d3a685 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 11:23:03 +0200 Subject: [PATCH 117/136] Restore stream reader un-examine state --- Slon.Tests/Pipelines/StreamPipeReaderTests.cs | 24 +++++++++++++++++++ Slon/Pipelines/StreamPipeReader.cs | 3 +-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Slon.Tests/Pipelines/StreamPipeReaderTests.cs b/Slon.Tests/Pipelines/StreamPipeReaderTests.cs index cae06d5..8087073 100644 --- a/Slon.Tests/Pipelines/StreamPipeReaderTests.cs +++ b/Slon.Tests/Pipelines/StreamPipeReaderTests.cs @@ -19,6 +19,30 @@ static async Task CreateBufferedReader(byte[] bytes) return reader; } + [TestMethod] + public async Task AdvanceTo_CanUnexamineBufferedData() + { + var reader = new DefaultStreamPipeReader( + new MemoryStream("Hello World"u8.ToArray(), writable: false), + new StreamPipeReaderOptions(bufferSize: 1024, useZeroByteReads: false), + supportCancelPending: false); + var first = await reader.ReadAsync(); + reader.AdvanceTo(first.Buffer.GetPosition(6), first.Buffer.End); + + // Examining the complete first grant forces the stream EOF probe. + var eof = await reader.ReadAsync(); + CollectionAssert.AreEqual("World"u8.ToArray(), eof.Buffer.ToArray()); + Assert.IsTrue(eof.IsCompleted); + + // Moving examined back to consumed must republish the existing suffix without another + // stream read, matching the .NET 10 PipeReader un-examine contract. + reader.AdvanceTo(eof.Buffer.Start, eof.Buffer.Start); + Assert.IsTrue(reader.TryRead(out var unexamined)); + CollectionAssert.AreEqual("World"u8.ToArray(), unexamined.Buffer.ToArray()); + reader.AdvanceTo(unexamined.Buffer.End); + await reader.CompleteAsync(); + } + [TestMethod] public async Task CopyToAsync_Stream_ConsumesSuccessfullyCopiedBufferedData() { diff --git a/Slon/Pipelines/StreamPipeReader.cs b/Slon/Pipelines/StreamPipeReader.cs index a311e0a..a05962c 100644 --- a/Slon/Pipelines/StreamPipeReader.cs +++ b/Slon/Pipelines/StreamPipeReader.cs @@ -75,8 +75,7 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami ThrowIfCompleted(); var bufferedBytes = Segments.BufferedBytes; var examinedBytes = Segments.AdvanceTo(consumed, examined); - if (examinedBytes == bufferedBytes) - ExaminedEverything = true; + ExaminedEverything = examinedBytes == bufferedBytes; } /// From 82e6137eea74d454221c0526ff262fb7f7d107a4 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 12:02:31 +0200 Subject: [PATCH 118/136] Release completed read grants at the idle edge --- Slon.Tests/Pg/BackendMessageStreamingTests.cs | 26 +++++++++++++++++ Slon/Pg/Protocol/BackendMessageContext.cs | 11 ++++++++ Slon/Pg/Protocol/PgClientProtocol.cs | 7 ++++- Slon/Pg/Protocol/PgDecoder.cs | 3 ++ Slon/Pg/Protocol/ProtocolReadPipe.cs | 28 +++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) diff --git a/Slon.Tests/Pg/BackendMessageStreamingTests.cs b/Slon.Tests/Pg/BackendMessageStreamingTests.cs index 7fa1622..3b23529 100644 --- a/Slon.Tests/Pg/BackendMessageStreamingTests.cs +++ b/Slon.Tests/Pg/BackendMessageStreamingTests.cs @@ -580,6 +580,32 @@ public async Task BeginningNewResultRetention_DropsPriorResultPrefixAtNextRead() await readPipe.DisposeAsync(); } + [TestMethod] + public async Task IdleRelease_UnexaminesAndReacquiresBufferedSuffix() + { + var completed = BackendMessageBytes(BackendType.ReadyForQuery, [(byte)'I']); + var suffix = BackendMessageBytes(BackendType.NotificationResponse, [1, 2, 3]); + var wire = new byte[completed.Length + suffix.Length]; + completed.CopyTo(wire, 0); + suffix.CopyTo(wire, completed.Length); + var pipe = new Pipe(); + var readPipe = new ProtocolReadPipe(pipe.Reader, + BackendMessageCursor.DefaultDataRowStreamingThreshold); + await pipe.Writer.WriteAsync(wire); + + Assert.IsTrue(await readPipe.MoveNextAsync(default)); + Assert.IsTrue(readPipe.TryMoveNext()); + Assert.AreEqual(BackendType.ReadyForQuery, readPipe.CurrentType); + + readPipe.ReleaseReadBufferAtIdle(); + + Assert.IsTrue(await readPipe.MoveNextAsync(default)); + Assert.IsTrue(readPipe.TryMoveNext()); + Assert.AreEqual(BackendType.NotificationResponse, readPipe.CurrentType); + await pipe.Writer.CompleteAsync(); + await readPipe.DisposeAsync(); + } + [TestMethod] public async Task BackendBodyReader_ExtendsPrefixThenSlides() { diff --git a/Slon/Pg/Protocol/BackendMessageContext.cs b/Slon/Pg/Protocol/BackendMessageContext.cs index e81a743..47e23d5 100644 --- a/Slon/Pg/Protocol/BackendMessageContext.cs +++ b/Slon/Pg/Protocol/BackendMessageContext.cs @@ -243,6 +243,17 @@ internal void RebaseCurrentMessageOffset() _currentMessageOffset = 0; } + internal bool TryGetCursorConsumedLength(out long consumedLength) + { + if (!_hasCursor) + { + consumedLength = 0; + return false; + } + consumedLength = _cursor.ConsumedLength; + return true; + } + public ReadOnlyMemory GetContiguousMemory( short token, ReadOnlyMemory source) { diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index 433176a..6f5181e 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1881,10 +1881,15 @@ internal void AssignCancellationBoundary(PgClientFlow flow, int window) internal void OnReleasing(PgClientFlow flow) { + // Draghi clears ActivatedFlow at the exact zero edge but retains the old activation turn + // through this callback. Null is therefore both the idle witness and an exclusive + // pre-release window in which no successor reader can activate. + var idle = ActivatedFlow is null; Decoder.EndResultBuffering(flow); + if (idle) + Decoder.ReleaseReadBufferAtIdle(); protocol._serverParameterState.CommitFlow(); ClearCancellationActivation(flow); - var idle = ActivatedFlow is null; protocol.OnFlowReleased(flow, poolFacing && idle); // Inner exclusive-scope subflows are not pool load units; only the outer pipeline reports // admission-to-retirement lifetimes to its host. diff --git a/Slon/Pg/Protocol/PgDecoder.cs b/Slon/Pg/Protocol/PgDecoder.cs index d67ab70..4d4cfb4 100644 --- a/Slon/Pg/Protocol/PgDecoder.cs +++ b/Slon/Pg/Protocol/PgDecoder.cs @@ -144,6 +144,9 @@ internal void EndResultBuffering(PgClientFlow owner) _pipe.EndResultRetention(); } + internal void ReleaseReadBufferAtIdle() + => _pipe.ReleaseReadBufferAtIdle(); + void ValidateResultBufferingOwner() { var owner = _resultBufferingOwner; diff --git a/Slon/Pg/Protocol/ProtocolReadPipe.cs b/Slon/Pg/Protocol/ProtocolReadPipe.cs index d458136..9362006 100644 --- a/Slon/Pg/Protocol/ProtocolReadPipe.cs +++ b/Slon/Pg/Protocol/ProtocolReadPipe.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Diagnostics; using System.IO.Pipelines; using Slon.Pipelines; @@ -446,6 +447,33 @@ public void EndResultRetention() _messageContext.ReleaseContiguousProjections(); } + // The retiring zero-edge owner calls this before its activation turn is released. End the + // current PipeReader grant at the parsed successor boundary so an idle connection cannot retain + // completed result storage. Any buffered suffix is deliberately un-examined and will be + // reacquired by the next ordinary read. + public void ReleaseReadBufferAtIdle() + { + Debug.Assert(!_retainsResult); + if (!_hasActiveRead || _pendingRead is not PendingRead.None + || _currentMessageLength > 0 + || !_messageContext.TryGetCursorConsumedLength(out var cursorConsumedLength)) + return; + + var consumedOffset = checked(_pendingCursorOffset + cursorConsumedLength); + var consumed = _activeBuffer.GetPosition(consumedOffset); + _messageContext.RetireCursor(); + reader.AdvanceTo(consumed, consumed); + _activeBuffer = default; + _examined = default; + _retainedStart = default; + _retainedOffset = 0; + _currentMessageOffset = 0; + _currentMessageLength = -1; + _pendingCursorOffset = 0; + _minimumReadSize = 0; + _hasActiveRead = false; + } + public void Dispose() { _messageContext.RetireCursor(); From 6215548af852f58e52569f83822eaff82b52f074 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 20:25:43 +0200 Subject: [PATCH 119/136] Allow the PantherTE comparison harness --- Slon/Slon.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Slon/Slon.csproj b/Slon/Slon.csproj index bac62fb..e6d4a7f 100644 --- a/Slon/Slon.csproj +++ b/Slon/Slon.csproj @@ -13,6 +13,8 @@ + + From 7e0b7d8523cdfb70745723b9bab78309b4409dcd Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Thu, 3 Sep 2026 20:32:49 +0200 Subject: [PATCH 120/136] Route Slon continuations through the ambient scheduler --- Slon/Pg/Protocol/CancellationCoordinator.cs | 10 ++++- Slon/Pg/Protocol/Flows/CommandFlow.cs | 16 +++++--- .../Flows/FlowCallerInteractionCore.cs | 5 ++- Slon/Pg/Protocol/PgClientProtocol.cs | 7 ++++ Slon/Pipelines/DelegatedPipelineScheduler.cs | 10 +++++ .../PromiseAsyncValueTaskMethodBuilder.cs | 5 ++- Slon/Threading/Scheduler.cs | 20 ++++++++++ Slon/Threading/SchedulingContext.cs | 38 +++++++++++++++++++ .../Tasks/Sources/ContinuationDispatcher.cs | 8 ++-- Slon/Transport/TransportConnection.cs | 2 + 10 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 Slon/Pipelines/DelegatedPipelineScheduler.cs create mode 100644 Slon/Threading/Scheduler.cs create mode 100644 Slon/Threading/SchedulingContext.cs diff --git a/Slon/Pg/Protocol/CancellationCoordinator.cs b/Slon/Pg/Protocol/CancellationCoordinator.cs index 9f7dad1..7568fe9 100644 --- a/Slon/Pg/Protocol/CancellationCoordinator.cs +++ b/Slon/Pg/Protocol/CancellationCoordinator.cs @@ -335,8 +335,14 @@ void StartDispatch(DispatchLease lease) // Sender delegates may execute arbitrary synchronous work before returning their ValueTask. // The queued callback claims physical start immediately before invocation, so neither a // dormant work item nor blocked sender can occupy the coordinator lock or deadline thread. - if (!ThreadPool.UnsafeQueueUserWorkItem(static state => state.Coordinator.InvokeDispatch(state.Lease), - (Coordinator: this, Lease: lease), preferLocal: false)) + if (!Slon.Threading.SchedulingContext.TrySubmitDetached( + static state => + { + var dispatch = ((CancellationCoordinator Coordinator, DispatchLease Lease))state!; + dispatch.Coordinator.InvokeDispatch(dispatch.Lease); + }, + (this, lease), + preferLocal: false)) _ = ObserveDispatchAsync(lease, new(CancelRequestState.NotSent)); } diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 8f2bb8a..fe39191 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -1043,9 +1043,11 @@ bool TryTakeOverDrain() NotifyDrainStarted(); // Decoder takeover does not imply consumer abandonment. Explicit cancellation and // graceful close also drain autonomously while retaining their consumer semantics. - ThreadPool.UnsafeQueueUserWorkItem(static state => - _ = new CommandFlowCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), - _ops.Flow); + Slon.Threading.SchedulingContext.SubmitDetached( + static state => + _ = new CommandFlowCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ops.Flow, + preferLocal: false); return true; } } @@ -1113,9 +1115,11 @@ void HandleReadTimeout(TimeoutException exception) Interlocked.Exchange(ref _state.Phase, PhaseDraining); RequestCancel(default, CommandExecutionCancellationScope.RemainingFlow, BackendCancellationTiming.Immediate, BackendCancellationTiming.AtReadFrontier); - ThreadPool.UnsafeQueueUserWorkItem(static state => - _ = new CommandFlowCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), - _ops.Flow); + Slon.Threading.SchedulingContext.SubmitDetached( + static state => + _ = new CommandFlowCore(TOps.Create((PgClientFlow)state!)).DrainAsync(), + _ops.Flow, + preferLocal: false); } void Drain() diff --git a/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs b/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs index 5328487..1ee22a5 100644 --- a/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs +++ b/Slon/Pg/Protocol/Flows/FlowCallerInteractionCore.cs @@ -147,7 +147,10 @@ static void Dispatch(Action continuation, bool useDedicatedDriver) { if (!useDedicatedDriver) { - ThreadPool.UnsafeQueueUserWorkItem(static s => ((Action)s!)(), (object)continuation); + Slon.Threading.SchedulingContext.SubmitDetached( + static state => ((Action)state!)(), + continuation, + preferLocal: false); return; } diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index 6f5181e..efcf713 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -305,9 +305,16 @@ public static PgClientProtocol Create(PgClientProtocolOptions protocolOptions) void Initialize(TransportConnection connection, Hosting hosting) { + PipelineScheduler? connectionScheduler = null; + if ((_options.ExecutionScheduler is null || _options.ActivationScheduler is null) + && connection.Scheduler is { } scheduler) + connectionScheduler = new DelegatedPipelineScheduler(scheduler); + _executionScheduler = _options.ExecutionScheduler + ?? connectionScheduler ?? PipelineScheduler.ThreadPool; _activationScheduler = _options.ActivationScheduler + ?? connectionScheduler ?? PipelineScheduler.ThreadPool; _connection = connection; diff --git a/Slon/Pipelines/DelegatedPipelineScheduler.cs b/Slon/Pipelines/DelegatedPipelineScheduler.cs new file mode 100644 index 0000000..9642d0f --- /dev/null +++ b/Slon/Pipelines/DelegatedPipelineScheduler.cs @@ -0,0 +1,10 @@ +using Draghi.Pipelining; +using Slon.Threading; + +namespace Slon.Pipelines; + +sealed class DelegatedPipelineScheduler(Scheduler scheduler) : PipelineScheduler +{ + public override void SubmitDetached(Action action, object? state, bool preferLocal = true) + => scheduler.SubmitDetached(action, state, preferLocal); +} diff --git a/Slon/Runtime/CompilerServices/PromiseAsyncValueTaskMethodBuilder.cs b/Slon/Runtime/CompilerServices/PromiseAsyncValueTaskMethodBuilder.cs index c347898..edab8f0 100644 --- a/Slon/Runtime/CompilerServices/PromiseAsyncValueTaskMethodBuilder.cs +++ b/Slon/Runtime/CompilerServices/PromiseAsyncValueTaskMethodBuilder.cs @@ -252,7 +252,10 @@ public void AwaitUnsafeOnCompleted( } catch (Exception ex) { - ThreadPool.QueueUserWorkItem(state => ((ExceptionDispatchInfo)state!).Throw(), ExceptionDispatchInfo.Capture(ex)); + Slon.Threading.SchedulingContext.Submit( + static state => ((ExceptionDispatchInfo)state!).Throw(), + ExceptionDispatchInfo.Capture(ex), + preferLocal: false); } } diff --git a/Slon/Threading/Scheduler.cs b/Slon/Threading/Scheduler.cs new file mode 100644 index 0000000..4016bbd --- /dev/null +++ b/Slon/Threading/Scheduler.cs @@ -0,0 +1,20 @@ +namespace Slon.Threading; + +abstract class Scheduler +{ + public abstract void SubmitDetached(Action action, object? state, bool preferLocal = true); +} + +sealed class DelegatedScheduler : Scheduler +{ + readonly Action, object?, bool> _submitDetached; + + public DelegatedScheduler(Action, object?, bool> submitDetached) + { + ArgumentNullException.ThrowIfNull(submitDetached); + _submitDetached = submitDetached; + } + + public override void SubmitDetached(Action action, object? state, bool preferLocal = true) + => _submitDetached(action, state, preferLocal); +} diff --git a/Slon/Threading/SchedulingContext.cs b/Slon/Threading/SchedulingContext.cs new file mode 100644 index 0000000..5ee0517 --- /dev/null +++ b/Slon/Threading/SchedulingContext.cs @@ -0,0 +1,38 @@ +namespace Slon.Threading; + +// Temporary host seam until the runtime provides the ambient scheduler lookup directly. +static class SchedulingContext +{ + static Action, object?, bool>? _submit; + static Action, object?, bool>? _submitDetached; + + public static void SetDispatch( + Action, object?, bool>? submit, + Action, object?, bool>? submitDetached) + { + _submit = submit; + _submitDetached = submitDetached; + } + + internal static void Submit(Action action, object? state, bool preferLocal) + { + var submit = Volatile.Read(ref _submit); + if (submit is null) + ThreadPool.QueueUserWorkItem(action, state, preferLocal); + else + submit(action, state, preferLocal); + } + + internal static void SubmitDetached(Action action, object? state, bool preferLocal) + => _ = TrySubmitDetached(action, state, preferLocal); + + internal static bool TrySubmitDetached(Action action, object? state, bool preferLocal) + { + var submit = Volatile.Read(ref _submitDetached); + if (submit is null) + return ThreadPool.UnsafeQueueUserWorkItem(action, state, preferLocal); + + submit(action, state, preferLocal); + return true; + } +} diff --git a/Slon/Threading/Tasks/Sources/ContinuationDispatcher.cs b/Slon/Threading/Tasks/Sources/ContinuationDispatcher.cs index a33309a..a84049a 100644 --- a/Slon/Threading/Tasks/Sources/ContinuationDispatcher.cs +++ b/Slon/Threading/Tasks/Sources/ContinuationDispatcher.cs @@ -53,7 +53,7 @@ public void SignalCompletion(bool runContinuationsAsynchronously) { if (runContinuationsAsynchronously) { - ThreadPool.UnsafeQueueUserWorkItem(continuation, _continuationState, preferLocal: true); + SchedulingContext.SubmitDetached(continuation, _continuationState, preferLocal: true); } else { @@ -136,11 +136,11 @@ public void OnCompleted(Action continuation, object? state, ValueTaskSo switch (capturedContext) { case null: - ThreadPool.UnsafeQueueUserWorkItem(continuation, state, preferLocal: true); + SchedulingContext.SubmitDetached(continuation, state, preferLocal: true); break; case ExecutionContext: - ThreadPool.QueueUserWorkItem(continuation, state, preferLocal: true); + SchedulingContext.Submit(continuation, state, preferLocal: true); break; default: @@ -204,7 +204,7 @@ static void InvokeWithContext(object capturedContext, Action continuati { try { - ThreadPool.QueueUserWorkItem(continuation, continuationState, preferLocal: true); + SchedulingContext.Submit(continuation, continuationState, preferLocal: true); } finally { diff --git a/Slon/Transport/TransportConnection.cs b/Slon/Transport/TransportConnection.cs index 4a05ae8..5904825 100644 --- a/Slon/Transport/TransportConnection.cs +++ b/Slon/Transport/TransportConnection.cs @@ -1,4 +1,5 @@ using System.IO.Pipelines; +using Slon.Threading; namespace Slon.Transport; @@ -30,6 +31,7 @@ public readonly struct ResumableWrite(ResumeSignal? signal, TimeSpan timeout) // The protocol completes both endpoints after all borrowed buffers have been returned. public abstract PipeReader Reader { get; } public abstract PipeWriter Writer { get; } + internal virtual Scheduler? Scheduler => null; // Classifies transport-specific exceptions that mean the established byte stream was lost. public virtual bool IsConnectionLost(Exception exception) => false; From 91a69629171efd81bfaeb2c732718d4b0fb30a18 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 20:15:43 +0200 Subject: [PATCH 121/136] Allow unsafe unified flow pooling experiment --- Slon/Pg/Protocol/PgClientFlow.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Slon/Pg/Protocol/PgClientFlow.cs b/Slon/Pg/Protocol/PgClientFlow.cs index 369cdbd..5018095 100644 --- a/Slon/Pg/Protocol/PgClientFlow.cs +++ b/Slon/Pg/Protocol/PgClientFlow.cs @@ -440,10 +440,10 @@ public void Reset() // rollover, a fail-loud TimeoutException at worst). If cancellation-aware flows become reusable, // that same reference-plus-stamp identity should be the cancellation coordinator's owner: a raw // reference cannot distinguish retained attribution from a later tenure whose window restarts at - // zero. Until the stamp lands, refuse to recycle a timeout-armed flow rather than let the race - // silently reappear. - if (EnableActivationTimeout) - ThrowHelper.ThrowInvalidOperation("Cannot pool a flow with EnableActivationTimeout: a recycled instance can be wrong-tenure-completed by a stale activation timeout. Implement generation-checked completion first."); + // zero. Production must refuse to recycle a timeout-armed flow until that stamp lands. + // EXPERIMENT ONLY: deliberately permit timeout-armed flows to be recycled so the unified + // flow's allocation-free throughput ceiling can be measured. This is not safe against a + // stale activation-timeout heartbeat completing a later tenure. _started = false; _completed = false; // Version bump per tenure. Cross-tenure completer staleness rests on the done -> torn-down From 28a4fd5df910b7c0865a6a797de1a9a2723995c9 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 01:31:30 +0200 Subject: [PATCH 122/136] Prototype tenure-safe pooled command flows --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 116 ++++++++++++++++++++++---- Slon/Pg/Protocol/PgClientProtocol.cs | 82 ++++++++++++++---- Slon/Slon.csproj | 2 +- 3 files changed, 170 insertions(+), 30 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index fe39191..73da9b7 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -73,6 +73,13 @@ internal struct CommandExecutionState internal Slon.Threading.Tasks.Sources.ManualResetValueTaskSourceCore PipelineTaskSource; internal CancellationToken FlowToken; internal CancellationTokenRegistration FlowRegistration; + // Scratch owned by the single active result consumer. Keeping it with the flow lets the + // resumable frames carry only their control state across suspension. + internal CancellationToken WindowToken; + // Every pre-consumed successor can suspend in FirstAsync simultaneously. Retain that frame with + // its reusable flow instead of competing for the builder's one-thread/one-core cache slots. + internal ValueTaskSourcePromise? FirstPromise; + internal bool RetainFirstPromise; internal CommandExecutionColdState? ColdState; internal FlowHandoffEvent? HandoffEvent; internal bool SyncHandoffClaimed; @@ -366,9 +373,14 @@ void OnActivationSettled(bool onExecutorStrand) return; } - if (IsCancelRequested) + var cancellationRequested = IsCancelRequested; + if (cancellationRequested) RequestBackendCancellation(); - var dispatchReady = _ops.IsAsyncAtDispatch && onExecutorStrand; + var context = _state.Context; + var stopping = context.StoppingToken.IsCancellationRequested; + var stoppingException = stopping ? context.FlowTerminationException : null; + var dispatchReady = (_ops.IsAsyncAtDispatch && onExecutorStrand) + || stopping || cancellationRequested; if (!CompleteReady(null, runContinuationsAsynchronously: dispatchReady)) { // Teardown released the consumer while this flow waited for its turn. Nothing reads the @@ -380,14 +392,14 @@ void OnActivationSettled(bool onExecutorStrand) // the in-flight store into the activated slot while that pass is being taken and miss it. // Recheck after publishing readiness: OnStopping arbitrates PhaseInitial against a consumer's // PhaseReading claim, so exactly one side owns the decoder and eventual pipeline completion. - if (_state.Context.StoppingToken.IsCancellationRequested) + if (stopping) { - OnStopping(_state.Context.FlowTerminationException); + OnStopping(stoppingException!); return; } // A cancel latched before activation may have released its caller already. The response // still has to reach RFQ, so drain it unless a consumer already owns the decoder. - if (IsCancelRequested) + if (cancellationRequested) TryTakeOverDrain(); } @@ -574,13 +586,15 @@ internal ValueTask MoveNextAsync(CancellationToken cancellationToken) if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseInitial) != PhaseInitial) continue; _state.CommandIndex = 0; - return FirstAsync(cancellationToken); + _state.WindowToken = cancellationToken; + return FirstAsync(); case PhaseResultReady: if (cancellationToken.IsCancellationRequested) return CancelBeforeRead(cancellationToken); if (Interlocked.CompareExchange(ref _state.Phase, PhaseReading, PhaseResultReady) != PhaseResultReady) continue; - return NextBatchAsync(cancellationToken); + RegisterCancellation(cancellationToken); + return NextBatchAsync(); case PhaseReading: return ValueTask.FromException( ThrowHelper.ThrowInvalidOperation("A read is already in progress on this flow.")); @@ -608,16 +622,72 @@ async ValueTask AwaitTakeoverAsync() throw Volatile.Read(ref _state.ColdState)?.TerminalException ?? ThrowHelper.ThrowInvalidOperation("The flow was disposed."); } + ValueTask FirstAsync() + { + var ready = new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version); + if (ready.IsCompletedSuccessfully) + { + _ = ready.Result; + return FirstAfterReadyAsync(); + } + if (!_state.RetainFirstPromise) + return AwaitReadyPooledAsync(ready); + var promise = _state.FirstPromise ??= new(); + using (PromiseAsyncValueTaskMethodBuilder.BeginCallScope(promise)) + return AwaitReadyRetainedAsync(ready); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] + async ValueTask AwaitReadyRetainedAsync(ValueTask ready) + { + try + { + await ready.ConfigureAwait(false); + } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + return await FirstAfterReadyAsync().ConfigureAwait(false); + } + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask FirstAsync(CancellationToken cancellationToken) + async ValueTask AwaitReadyPooledAsync(ValueTask ready) + { + try + { + await ready.ConfigureAwait(false); + } + catch (TimeoutException ex) + { + HandleReadTimeout(ex); + throw; + } + catch (Exception ex) + { + FaultFromOwner(ex); + throw; + } + return await FirstAfterReadyAsync().ConfigureAwait(false); + } + + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] + async ValueTask FirstAfterReadyAsync() { Exception? deliver; try { - await new ValueTask((IValueTaskSource)_ops.Flow, _state.ReadySource.Version).ConfigureAwait(false); Debug.Assert(!_state.ConsumerDetached); - RegisterCancellation(cancellationToken); + RegisterCancellation(TakeWindowToken()); var result = IsSinglePublishedCommand ? await ReadResultAsync().ConfigureAwait(false) : await ReadNextPublishedResultAsync().ConfigureAwait(false); @@ -662,11 +732,10 @@ async ValueTask FirstAsync(CancellationToken cancellationToken) [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] - async ValueTask NextBatchAsync(CancellationToken cancellationToken) + async ValueTask NextBatchAsync() { try { - RegisterCancellation(cancellationToken); var result = _state.Current!; await _state.Context.GetProtocolStatic() .ResultMessageEnumerator.DisposeAsync().ConfigureAwait(false); @@ -729,6 +798,13 @@ async ValueTask NextBatchAsync(CancellationToken cancellationToken) } } + CancellationToken TakeWindowToken() + { + var token = _state.WindowToken; + _state.WindowToken = default; + return token; + } + [RuntimeAsyncMethodGeneration(false)] [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask ReadNextPublishedResultAsync() @@ -830,6 +906,8 @@ ValueTask ReadResultAsync() : ReadUnpreparedResultAsync(); } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask ReadPreparationResultAsync() { ref readonly var command = ref _state.Commands.ItemRef(_state.CommandIndex); @@ -841,6 +919,8 @@ async ValueTask ReadPreparationResultAsync() _state.CommandIndex, preparation.Item1, preparation.Item3, preparation.Item2); } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask ReadPreparedResultAsync() { // Prepared commands with a known description have the compact BindComplete -> @@ -871,6 +951,8 @@ async ValueTask ReadPreparedResultAsync() return InitializeResult(_state.CommandIndex, error, null); } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] async ValueTask ReadUnpreparedResultAsync() { ref readonly var command = ref _state.Commands.ItemRef(_state.CommandIndex); @@ -964,11 +1046,13 @@ void SkipDiscardedCommands() _state.ReadFlowRfq = false; } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask ReadRfqAsync() { var message = await _state.Context.Decoder.GetNextAsync().ConfigureAwait(false); - if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } rfqError) - PgErrorException.Throw(rfqError); + if (message.EnsureExpectedOrError(PgTypes.BackendType.ReadyForQuery) is { } error) + PgErrorException.Throw(error); } void ReadRfq() @@ -978,6 +1062,8 @@ void ReadRfq() PgErrorException.Throw(rfqError); } + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] async ValueTask CompleteBatchAsync() { if (_state.ReadFlowRfq) @@ -1516,6 +1602,8 @@ internal void OnReset() _state.DrainStarted = 0; _state.FlowToken = default; _state.FlowRegistration = default; + _state.WindowToken = default; + _state.RetainFirstPromise = true; _state.ColdState = null; _state.SyncHandoffClaimed = false; _state.HandoffEvent?.ResetInteraction(); diff --git a/Slon/Pg/Protocol/PgClientProtocol.cs b/Slon/Pg/Protocol/PgClientProtocol.cs index efcf713..3048674 100644 --- a/Slon/Pg/Protocol/PgClientProtocol.cs +++ b/Slon/Pg/Protocol/PgClientProtocol.cs @@ -1146,28 +1146,47 @@ public ValueTask HeartbeatAsync(TimeSpan elapsed) void PropagateFlowHeartbeat(TimeSpan period) { var control = FlowControl; + control.BeginHeartbeatObservation(); try { - _source.OnActivationHeartbeat(period); - } - catch (Exception ex) - { - SlonLogMessages.UnobservedCallbackException( - _logger, ex, "the source heartbeat callback"); - } - - foreach (var flow in GetFlows()) - { - try + // Backlog observation will get its own bounded source frontier. Holding the gate for + // this phase is sufficient for the experiment and free when the backlog is empty. + lock (control.HeartbeatObservationLock) { - flow.GetExecutionControl(control).OnHeartbeat(period); + try + { + _source.OnActivationHeartbeat(period); + } + catch (Exception ex) + { + SlonLogMessages.UnobservedCallbackException( + _logger, ex, "the source heartbeat callback"); + } } - catch (Exception ex) + + var flows = _pipeline.GetEnumerator(out var frontier); + while (flows.MoveNext()) { - SlonLogMessages.UnobservedCallbackException( - _logger, ex, "a flow heartbeat callback"); + lock (control.HeartbeatObservationLock) + { + if (frontier.IsRetired(flows.Position)) + continue; + try + { + flows.Current.GetExecutionControl(control).OnHeartbeat(period); + } + catch (Exception ex) + { + SlonLogMessages.UnobservedCallbackException( + _logger, ex, "a flow heartbeat callback"); + } + } } } + finally + { + control.EndHeartbeatObservation(); + } } void PropagateFlowTermination() @@ -1220,6 +1239,18 @@ public Policy(PgClientProtocol protocol, Control control, ExclusiveScopeState? l [MethodImpl(MethodImplOptions.AggressiveInlining)] public void CompleteItem(PgClientFlow item, Exception? exception) + { + if (_control.IsHeartbeatObserving) + { + lock (_control.HeartbeatObservationLock) + CompleteItemCore(item, exception); + return; + } + CompleteItemCore(item, exception); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void CompleteItemCore(PgClientFlow item, Exception? exception) { if (exception is PgClientClosedException && _control.ClosedException is not null) exception = _control.FlowTerminationException; @@ -1554,6 +1585,27 @@ internal sealed class Control(PgClientProtocol protocol, bool poolFacing) : IProtocolStatic, IProtocolStatic { + readonly Lock _heartbeatObservationLock = new(); + int _heartbeatObserving; + + internal Lock HeartbeatObservationLock => _heartbeatObservationLock; + internal bool IsHeartbeatObserving => Volatile.Read(ref _heartbeatObserving) != 0; + + internal void BeginHeartbeatObservation() + { + lock (_heartbeatObservationLock) + { + Debug.Assert(_heartbeatObserving == 0); + Volatile.Write(ref _heartbeatObserving, 1); + } + } + + internal void EndHeartbeatObservation() + { + lock (_heartbeatObservationLock) + Volatile.Write(ref _heartbeatObserving, 0); + } + // The pipeline whose slots this Control reads, bound right after that pipeline is created. The // outer (pool-facing) Control reads the protocol's own pipeline; an exclusive flow's inner // Control reads its inner pipeline - both through the same IPipelineSlots handle, so any diff --git a/Slon/Slon.csproj b/Slon/Slon.csproj index e6d4a7f..083ddc8 100644 --- a/Slon/Slon.csproj +++ b/Slon/Slon.csproj @@ -23,7 +23,7 @@ - + From e0d061d3b702bbfb1bd5685f27284fcff06abf72 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 01:32:08 +0200 Subject: [PATCH 123/136] Retain activation waiter only after flow reuse --- Slon/Pg/Protocol/Flows/CommandFlow.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Slon/Pg/Protocol/Flows/CommandFlow.cs b/Slon/Pg/Protocol/Flows/CommandFlow.cs index 73da9b7..cbcf656 100644 --- a/Slon/Pg/Protocol/Flows/CommandFlow.cs +++ b/Slon/Pg/Protocol/Flows/CommandFlow.cs @@ -79,7 +79,6 @@ internal struct CommandExecutionState // Every pre-consumed successor can suspend in FirstAsync simultaneously. Retain that frame with // its reusable flow instead of competing for the builder's one-thread/one-core cache slots. internal ValueTaskSourcePromise? FirstPromise; - internal bool RetainFirstPromise; internal CommandExecutionColdState? ColdState; internal FlowHandoffEvent? HandoffEvent; internal bool SyncHandoffClaimed; @@ -630,9 +629,8 @@ ValueTask FirstAsync() _ = ready.Result; return FirstAfterReadyAsync(); } - if (!_state.RetainFirstPromise) + if (_state.FirstPromise is not { } promise) return AwaitReadyPooledAsync(ready); - var promise = _state.FirstPromise ??= new(); using (PromiseAsyncValueTaskMethodBuilder.BeginCallScope(promise)) return AwaitReadyRetainedAsync(ready); } @@ -1603,7 +1601,7 @@ internal void OnReset() _state.FlowToken = default; _state.FlowRegistration = default; _state.WindowToken = default; - _state.RetainFirstPromise = true; + _state.FirstPromise ??= new(); _state.ColdState = null; _state.SyncHandoffClaimed = false; _state.HandoffEvent?.ResetInteraction(); From 32dbf23197694a083140c4f05905d4ff5731eb15 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 08:56:21 +0200 Subject: [PATCH 124/136] Shrink runtime async pipe read suspension state --- Slon/Pipelines/StreamPipeReader.cs | 73 ++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/Slon/Pipelines/StreamPipeReader.cs b/Slon/Pipelines/StreamPipeReader.cs index a05962c..59923da 100644 --- a/Slon/Pipelines/StreamPipeReader.cs +++ b/Slon/Pipelines/StreamPipeReader.cs @@ -385,7 +385,7 @@ protected ValueTask ReadAsyncCore(int minimumSize, CancellationToken try { #endif - return ReadAsyncCore(minimumSize, PendingReadTokenSource, cancellationToken); + return StartReadAsync(minimumSize, PendingReadTokenSource, cancellationToken); #if !NET11_0_OR_GREATER } finally @@ -394,27 +394,52 @@ protected ValueTask ReadAsyncCore(int minimumSize, CancellationToken } #endif -#if !NET11_0_OR_GREATER - [RuntimeAsyncMethodGeneration(false)] - [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] -#endif - async ValueTask ReadAsyncCore(int minimumSize, + ValueTask StartReadAsync(int minimumSize, AutoResetCancellationTokenSource? tokenSource, CancellationToken cancellationToken) { - // Cancellation token was already checked before getting here. if (!TryStartRead()) ThrowAlreadyReading(); CancellationTokenRegistration reg = default; - CancellationToken token; - if (tokenSource is { } src) + var registered = false; + try { - if (cancellationToken.CanBeCanceled) - reg = src.UnsafeRegister(cancellationToken); - token = src.Token; + CancellationToken token; + if (tokenSource is { } src) + { + if (cancellationToken.CanBeCanceled) + { + registered = true; + reg = src.UnsafeRegister(cancellationToken); + } + token = src.Token; + } + else + { + token = cancellationToken; + } + + var read = ReadLoopAsync( + minimumSize, token, cancellationToken, endStartedRead: !registered); + return registered ? CompleteRegisteredReadAsync(read, reg) : read; } - else - token = cancellationToken; + catch + { + if (registered) + reg.Dispose(); + EndStartedRead(); + throw; + } + } + +#if !NET11_0_OR_GREATER + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PromiseAsyncValueTaskMethodBuilder<>))] +#endif + async ValueTask ReadLoopAsync( + int minimumSize, CancellationToken token, CancellationToken cancellationToken, + bool endStartedRead) + { try { if (Segments.BufferedBytes is 0 && UseZeroByteReads) @@ -448,10 +473,28 @@ async ValueTask ReadAsyncCore(int minimumSize, throw; } finally + { + if (endStartedRead) + EndStartedRead(); + } + } + +#if !NET11_0_OR_GREATER + [RuntimeAsyncMethodGeneration(false)] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] +#endif + async ValueTask CompleteRegisteredReadAsync( + ValueTask read, CancellationTokenRegistration registration) + { + try + { + return await read.ConfigureAwait(false); + } + finally { try { - await reg.DisposeAsync().ConfigureAwait(false); + await registration.DisposeAsync().ConfigureAwait(false); } finally { From 8f7d68476e63c46e968234a3350d1bc8ee2576f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:41:53 -0700 Subject: [PATCH 125/136] Add Slon fortunes benchmarks Port the Minimal API and low-level Platform fortunes workloads with comparable Slon and Npgsql PostgreSQL paths, plus multi-source Crank scenarios. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Slon.Fortunes.Minimal/Fortune.cs | 17 ++ .../Slon.Fortunes.Minimal/FortuneDatabase.cs | 197 +++++++++++++++ .../Slon.Fortunes.Minimal/Program.cs | 36 +++ .../Slon.Fortunes.Minimal/README.md | 33 +++ .../Slon.Fortunes.Minimal.csproj | 19 ++ .../Templates/Fortunes.cshtml | 2 + .../Templates/_ViewImports.cshtml | 9 + .../minimal-fortunes.benchmarks.yml | 88 +++++++ .../BenchmarkApplication.HttpConnection.cs | 163 ++++++++++++ .../BenchmarkApplication.cs | 119 +++++++++ .../BufferExtensions.cs | 56 +++++ .../Slon.Fortunes.Platform/BufferWriter.cs | 133 ++++++++++ .../ChunkedPipeWriter.cs | 235 ++++++++++++++++++ .../Slon.Fortunes.Platform/DateHeader.cs | 53 ++++ .../Slon.Fortunes.Platform/Fortune.cs | 17 ++ .../Slon.Fortunes.Platform/FortuneDatabase.cs | 189 ++++++++++++++ .../Slon.Fortunes.Platform/HttpApplication.cs | 28 +++ .../Slon.Fortunes.Platform/IHttpConnection.cs | 18 ++ .../Slon.Fortunes.Platform/Program.cs | 49 ++++ .../Slon.Fortunes.Platform/README.md | 32 +++ .../Slon.Fortunes.Platform.csproj | 20 ++ .../Templates/Fortunes.cshtml | 2 + .../Templates/_ViewImports.cshtml | 9 + .../platform-fortunes.benchmarks.yml | 88 +++++++ Slon.slnx | 2 + 25 files changed, 1614 insertions(+) create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/README.md create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/_ViewImports.cshtml create mode 100644 Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.HttpConnection.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/BufferExtensions.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/BufferWriter.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/ChunkedPipeWriter.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/DateHeader.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/HttpApplication.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/IHttpConnection.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/README.md create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/Templates/_ViewImports.cshtml create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs new file mode 100644 index 0000000..945e3e2 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs @@ -0,0 +1,17 @@ +namespace Slon.Fortunes.Minimal; + +public readonly struct Fortune : IComparable +{ + public Fortune(int id, string message) + { + Id = id; + Message = message; + } + + public int Id { get; } + + public string Message { get; } + + public int CompareTo(Fortune other) => + StringComparer.Ordinal.Compare(Message, other.Message); +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs new file mode 100644 index 0000000..73f91cd --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs @@ -0,0 +1,197 @@ +using System.Globalization; +using System.Net; +using Npgsql; + +namespace Slon.Fortunes.Minimal; + +internal abstract class FortuneDatabase : IAsyncDisposable +{ + protected const string Query = "SELECT id, message FROM fortune"; + private const string AdditionalFortune = "Additional fortune added at request time."; + + public abstract ValueTask DisposeAsync(); + + public abstract ValueTask> LoadAsync( + CancellationToken cancellationToken); + + public static ValueTask CreateAsync(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + + var database = RequiredDatabase(configuration["DATABASE"]); + var driver = RequiredDriver(configuration["DRIVER"]); + var connectionString = RequiredConnectionString(configuration["CONNECTION_STRING"]); + var connectionCount = PositiveSetting(configuration, "DATABASE_CONNECTIONS"); + + return (database, driver) switch + { + ("postgresql", "slon") => SlonFortuneDatabase.CreateAsync( + connectionString, + connectionCount, + PositiveSetting(configuration, "SLON_PIPELINING")), + ("postgresql", "npgsql") => ValueTask.FromResult( + new NpgsqlFortuneDatabase(connectionString, connectionCount)), + _ => throw new InvalidOperationException("The database selection is invalid."), + }; + } + + protected static List Complete(List fortunes) + { + fortunes.Add(new Fortune(0, AdditionalFortune)); + fortunes.Sort(); + return fortunes; + } + + private static string RequiredDatabase(string? value) + { + var database = RequiredValue("DATABASE", value); + return database == "postgresql" + ? database + : throw new InvalidOperationException("DATABASE must be 'postgresql'."); + } + + private static string RequiredDriver(string? value) + { + var driver = RequiredValue("DRIVER", value); + return driver is "slon" or "npgsql" + ? driver + : throw new InvalidOperationException("DRIVER must be 'slon' or 'npgsql'."); + } + + private static string RequiredConnectionString(string? value) => + string.IsNullOrWhiteSpace(value) + ? throw new InvalidOperationException("CONNECTION_STRING is required.") + : value; + + private static string RequiredValue(string name, string? value) => + string.IsNullOrWhiteSpace(value) + ? throw new InvalidOperationException($"{name} is required.") + : value.Trim().ToLowerInvariant(); + + private static int PositiveSetting(IConfiguration configuration, string name) + { + var value = configuration[name] ?? + throw new InvalidOperationException($"{name} is required."); + + return int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) && + parsed > 0 + ? parsed + : throw new InvalidOperationException($"{name} must be a positive integer."); + } +} + +internal sealed class SlonFortuneDatabase : FortuneDatabase +{ + private readonly SlonDataSource _dataSource; + private readonly SlonCommand _command; + + private SlonFortuneDatabase(SlonDataSource dataSource, SlonCommand command) + { + _dataSource = dataSource; + _command = command; + } + + public static async ValueTask CreateAsync( + string connectionString, + int connectionCount, + int pipeliningLimit) + { + var builder = new NpgsqlConnectionStringBuilder(connectionString); + var dataSource = new SlonDataSource(new SlonDataSourceOptions + { + EndPoint = new DnsEndPoint( + RequiredPostgreSqlValue("Host", builder.Host), + builder.Port), + Database = RequiredPostgreSqlValue("Database", builder.Database), + Username = RequiredPostgreSqlValue("Username", builder.Username), + Password = builder.Password, + PoolSize = connectionCount, + MaxInFlightOperationsPerWire = pipeliningLimit, + Ssl = new PostgreSqlSslOptions + { + Mode = PostgreSqlSslMode.Disable, + }, + }); + + try + { + var command = dataSource.CreateCommand(Query); + try + { + await command.PrepareAsync(); + return new SlonFortuneDatabase(dataSource, command); + } + catch + { + await command.DisposeAsync(); + throw; + } + } + catch + { + await dataSource.DisposeAsync(); + throw; + } + } + + public override async ValueTask> LoadAsync( + CancellationToken cancellationToken) + { + await using var reader = await _command.ExecuteReaderAsync(cancellationToken); + List fortunes = []; + while (await reader.ReadAsync(cancellationToken)) + { + fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); + } + + return Complete(fortunes); + } + + public override async ValueTask DisposeAsync() + { + try + { + await _command.DisposeAsync(); + } + finally + { + await _dataSource.DisposeAsync(); + } + } + + private static string RequiredPostgreSqlValue(string name, string? value) => + string.IsNullOrWhiteSpace(value) + ? throw new InvalidOperationException($"PostgreSQL {name} is required.") + : value; +} + +internal sealed class NpgsqlFortuneDatabase : FortuneDatabase +{ + private readonly NpgsqlDataSource _dataSource; + + public NpgsqlFortuneDatabase(string connectionString, int connectionCount) + { + var builder = new NpgsqlConnectionStringBuilder(connectionString) + { + MaxPoolSize = connectionCount, + }; + _dataSource = new NpgsqlSlimDataSourceBuilder(builder.ConnectionString).Build(); + } + + public override async ValueTask> LoadAsync( + CancellationToken cancellationToken) + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var command = new NpgsqlCommand(Query, connection); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + List fortunes = []; + while (await reader.ReadAsync(cancellationToken)) + { + fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); + } + + return Complete(fortunes); + } + + public override ValueTask DisposeAsync() => _dataSource.DisposeAsync(); +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs new file mode 100644 index 0000000..802d4a3 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs @@ -0,0 +1,36 @@ +using System.Text.Encodings.Web; +using System.Text.Unicode; +using Slon.Fortunes.Minimal; +using Slon.Fortunes.Minimal.Templates; + +var builder = WebApplication.CreateBuilder(args); + +builder.Logging.ClearProviders(); + +await using var database = await FortuneDatabase.CreateAsync(builder.Configuration); +builder.Services.AddSingleton(CreateHtmlEncoder()); + +await using var app = builder.Build(); + +app.MapGet( + "/fortunes", + async (HtmlEncoder htmlEncoder, CancellationToken cancellationToken) => + { + var template = Fortunes.Create(await database.LoadAsync(cancellationToken)); + template.HtmlEncoder = htmlEncoder; + return template; + }); + +app.Lifetime.ApplicationStarted.Register(static () => Console.WriteLine("Application started.")); + +await app.RunAsync(); + +static HtmlEncoder CreateHtmlEncoder() +{ + var settings = new TextEncoderSettings( + UnicodeRanges.BasicLatin, + UnicodeRanges.Katakana, + UnicodeRanges.Hiragana); + settings.AllowCharacter('\u2014'); + return HtmlEncoder.Create(settings); +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md new file mode 100644 index 0000000..2efedde --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -0,0 +1,33 @@ +# Slon Minimal APIs Fortunes + +This standalone Minimal APIs benchmark exposes `GET /fortunes`. Every request loads all +`fortune` rows, appends the standard request-time fortune, sorts by message, and renders the +HTML response with RazorSlices HTML encoding. + +## Selection + +Set the following configuration values as environment variables or equivalent .NET configuration: + +| Setting | Values | +| --- | --- | +| `DATABASE` | `postgresql` | +| `DRIVER` | `slon` or `npgsql` | +| `CONNECTION_STRING` | PostgreSQL connection string | +| `DATABASE_CONNECTIONS` | Positive fixed pool size | +| `SLON_PIPELINING` | Positive per-wire in-flight limit for Slon | + +Invalid, unsupported, or missing selections fail application startup with an explicit error. +The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an +unmerged branch. + +## Driver strategies + +Slon uses a fixed-size `SlonDataSource` and one data-source-bound command prepared at startup. +The prepared command is reused concurrently and pipelines requests across the configured +connections. Npgsql uses a slim data source and a command bound to each leased connection. +Both drivers materialize messages with `GetString`, append and ordinally sort the same string +model, and render the same RazorSlices string template for a fair comparison. + +The Crank configuration uses two fewer Slon connections than database cores and 256 Npgsql +connections; Npgsql needs the additional in-flight operations to hide network and query latency. +`SLON_PIPELINING` is set to 16. diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj new file mode 100644 index 0000000..e6ff620 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml b/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml new file mode 100644 index 0000000..39a4cb9 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml @@ -0,0 +1,2 @@ +@inherits RazorSlice> +Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message
diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/_ViewImports.cshtml b/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/_ViewImports.cshtml new file mode 100644 index 0000000..2fb29eb --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/_ViewImports.cshtml @@ -0,0 +1,9 @@ +@inherits RazorSlice + +@using System.Globalization +@using Microsoft.AspNetCore.Razor +@using RazorSlices +@using Slon.Fortunes.Minimal + +@tagHelperPrefix __disable_tagHelpers__: +@removeTagHelper *, Microsoft.AspNetCore.Mvc.Razor diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml b/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml new file mode 100644 index 0000000..88a7858 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml @@ -0,0 +1,88 @@ +imports: + - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml + - https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/aspnet.profiles.standard.yml + +variables: + serverPort: 5000 + npgsqlConnections: 256 + # Override this value to benchmark an unmerged Slon branch or commit. + branchOrCommit: main + +jobs: + minimal-postgresql-slon: + sources: + Slon: + repository: https://github.com/draghidev/slon.git + branchOrCommit: "{{branchOrCommit}}" + Draghi: + repository: https://github.com/draghidev/pipelining.git + branchOrCommit: main + project: Slon/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj + framework: net10.0 + patchReferences: true + readyStateText: Application started. + arguments: "--urls {{serverScheme}}://{{serverAddress}}:{{serverPort}}" + variables: + serverScheme: http + environmentVariables: + DATABASE: postgresql + DRIVER: slon + CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass" + DATABASE_CONNECTIONS: "{{ cores | minus: 2 }}" + SLON_PIPELINING: 16 + + minimal-postgresql-npgsql: + sources: + Slon: + repository: https://github.com/draghidev/slon.git + branchOrCommit: "{{branchOrCommit}}" + Draghi: + repository: https://github.com/draghidev/pipelining.git + branchOrCommit: main + project: Slon/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj + framework: net10.0 + patchReferences: true + readyStateText: Application started. + arguments: "--urls {{serverScheme}}://{{serverAddress}}:{{serverPort}}" + variables: + serverScheme: http + environmentVariables: + DATABASE: postgresql + DRIVER: npgsql + CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass;Maximum Pool Size={{ npgsqlConnections }};No Reset On Close=true;Enlist=false;Max Auto Prepare=4" + DATABASE_CONNECTIONS: "{{ npgsqlConnections }}" + + postgresql: + source: + repository: https://github.com/TechEmpower/FrameworkBenchmarks.git + branchOrCommit: master + dockerFile: toolset/databases/postgres/postgres.dockerfile + dockerImageName: postgres_te + dockerContextDirectory: toolset/databases/postgres + readyStateText: ready to accept connections + noClean: true + +scenarios: + minimal-postgresql-slon: + db: + job: postgresql + application: + job: minimal-postgresql-slon + load: + job: wrk + variables: + presetHeaders: html + path: /fortunes + connections: 512 + + minimal-postgresql-npgsql: + db: + job: postgresql + application: + job: minimal-postgresql-npgsql + load: + job: wrk + variables: + presetHeaders: html + path: /fortunes + connections: 512 diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.HttpConnection.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.HttpConnection.cs new file mode 100644 index 0000000..001fa74 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.HttpConnection.cs @@ -0,0 +1,163 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +using System.Buffers; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Text.Encodings.Web; +using System.Text.Unicode; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; + +namespace Slon.Fortunes.Platform; + +public sealed partial class BenchmarkApplication : IHttpConnection +{ + private State _state; + + public CancellationToken ConnectionClosed { get; set; } + + public PipeReader Reader { get; set; } = null!; + + public PipeWriter Writer { get; set; } = null!; + + private HtmlEncoder HtmlEncoder { get; } = CreateHtmlEncoder(); + + private HttpParser Parser { get; } = new(); + + public async Task ExecuteAsync() + { + try + { + await ProcessRequestsAsync(); + Reader.Complete(); + } + catch (Exception ex) + { + Reader.Complete(ex); + } + finally + { + Writer.Complete(); + } + } + + private async Task ProcessRequestsAsync() + { + while (true) + { + var readResult = await Reader.ReadAsync(); + var buffer = readResult.Buffer; + var isCompleted = readResult.IsCompleted; + + if (buffer.IsEmpty && isCompleted) + { + return; + } + + while (true) + { + ParseHttpRequest(ref buffer, isCompleted); + + if (_state == State.Body) + { + await ProcessRequestAsync(); + _state = State.StartLine; + + if (!buffer.IsEmpty) + { + continue; + } + } + + Reader.AdvanceTo(buffer.Start, buffer.End); + break; + } + + await Writer.FlushAsync(); + } + } + + private void ParseHttpRequest(ref ReadOnlySequence buffer, bool isCompleted) + { + var reader = new SequenceReader(buffer); + var state = _state; + if (state == State.StartLine && + Parser.ParseRequestLine(new ParsingAdapter(this), ref reader)) + { + state = State.Headers; + } + + if (state == State.Headers && + Parser.ParseHeaders(new ParsingAdapter(this), ref reader)) + { + state = State.Body; + } + + if (state != State.Body && isCompleted) + { + throw new InvalidOperationException("Unexpected end of data!"); + } + + _state = state; + buffer = state == State.Body + ? buffer.Slice(reader.Position, 0) + : buffer.Slice(reader.Position); + } + + private static HtmlEncoder CreateHtmlEncoder() + { + var settings = new TextEncoderSettings( + UnicodeRanges.BasicLatin, + UnicodeRanges.Katakana, + UnicodeRanges.Hiragana); + settings.AllowCharacter('\u2014'); + return HtmlEncoder.Create(settings); + } + + public void OnStaticIndexedHeader(int index) + { + } + + public void OnStaticIndexedHeader(int index, ReadOnlySpan value) + { + } + + public void OnHeader(ReadOnlySpan name, ReadOnlySpan value) + { + } + + public void OnHeadersComplete(bool endStream) + { + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ReturnChunkedWriter(ChunkedPipeWriter writer) => ChunkedWriterPool.Return(writer); + + private enum State + { + StartLine, + Headers, + Body, + } + + private readonly struct ParsingAdapter(BenchmarkApplication requestHandler) : + IHttpRequestLineHandler, + IHttpHeadersHandler + { + public void OnStaticIndexedHeader(int index) => requestHandler.OnStaticIndexedHeader(index); + + public void OnStaticIndexedHeader(int index, ReadOnlySpan value) => + requestHandler.OnStaticIndexedHeader(index, value); + + public void OnHeader(ReadOnlySpan name, ReadOnlySpan value) => + requestHandler.OnHeader(name, value); + + public void OnHeadersComplete(bool endStream) => requestHandler.OnHeadersComplete(endStream); + + public void OnStartLine( + HttpVersionAndMethod versionAndMethod, + TargetOffsetPathLength targetPath, + Span startLine) => + requestHandler.OnStartLine(versionAndMethod, targetPath, startLine); + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs new file mode 100644 index 0000000..4d25db7 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs @@ -0,0 +1,119 @@ +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; +using Microsoft.Extensions.ObjectPool; +using RazorSlices; + +namespace Slon.Fortunes.Platform; + +public sealed partial class BenchmarkApplication +{ + private static readonly DefaultObjectPool ChunkedWriterPool = + new(new ChunkedWriterObjectPolicy()); + + private RequestType _requestType; + + internal static FortuneDatabase Database { get; set; } = null!; + + public void OnStartLine( + HttpVersionAndMethod versionAndMethod, + TargetOffsetPathLength targetPath, + Span startLine) + { + _requestType = versionAndMethod.Method == + Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http.HttpMethod.Get && + startLine.Slice(targetPath.Offset, targetPath.Length).SequenceEqual("/fortunes"u8) + ? RequestType.Fortunes + : RequestType.NotRecognized; + } + + private Task ProcessRequestAsync() => _requestType switch + { + RequestType.Fortunes => RenderDatabaseAsync(), + _ => OutputEmptyAsync(Writer), + }; + + private async Task RenderDatabaseAsync() + { + var template = Templates.Fortunes.Create( + await Database.LoadAsync(ConnectionClosed)); + await OutputFortunesAsync(Writer, template); + } + + private ValueTask OutputFortunesAsync( + PipeWriter pipeWriter, + RazorSlice template) + { + var chunkedWriter = StartResponse(pipeWriter); + var renderTask = template.RenderAsync(chunkedWriter, HtmlEncoder); + if (renderTask.IsCompletedSuccessfully) + { + renderTask.GetAwaiter().GetResult(); + EndTemplateRendering(chunkedWriter, template); + return ValueTask.CompletedTask; + } + + return AwaitTemplateRenderTask(renderTask, chunkedWriter, template); + } + + private static Task OutputEmptyAsync(PipeWriter pipeWriter) + { + var writer = StartResponse(pipeWriter); + writer.Complete(); + ReturnChunkedWriter(writer); + return Task.CompletedTask; + } + + private static ChunkedPipeWriter StartResponse(PipeWriter pipeWriter) + { + var preamble = + "HTTP/1.1 200 OK\r\nServer: K\r\nContent-Type: text/html; charset=utf-8\r\nTransfer-Encoding: chunked"u8; + var headersLength = preamble.Length + DateHeader.HeaderBytes.Length; + var headersSpan = pipeWriter.GetSpan(headersLength); + preamble.CopyTo(headersSpan); + DateHeader.HeaderBytes.CopyTo(headersSpan[preamble.Length..]); + pipeWriter.Advance(headersLength); + + var writer = ChunkedWriterPool.Get(); + writer.SetOutput(pipeWriter, 2048); + return writer; + } + + private static async ValueTask AwaitTemplateRenderTask( + ValueTask renderTask, + ChunkedPipeWriter chunkedWriter, + RazorSlice template) + { + await renderTask; + EndTemplateRendering(chunkedWriter, template); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EndTemplateRendering( + ChunkedPipeWriter chunkedWriter, + RazorSlice template) + { + chunkedWriter.Complete(); + ReturnChunkedWriter(chunkedWriter); + template.Dispose(); + } + + private sealed class ChunkedWriterObjectPolicy : + IPooledObjectPolicy + { + public ChunkedPipeWriter Create() => new(); + + public bool Return(ChunkedPipeWriter writer) + { + writer.Reset(); + return true; + } + } + + private enum RequestType + { + NotRecognized, + Fortunes, + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/BufferExtensions.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/BufferExtensions.cs new file mode 100644 index 0000000..55ba425 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/BufferExtensions.cs @@ -0,0 +1,56 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; + +namespace Slon.Fortunes.Platform; + +public static class BufferExtensions +{ + private const int MaxULongByteLength = 20; + + [ThreadStatic] + private static byte[]? s_numericBytesScratch; + + internal static void WriteUtf8String(ref this BufferWriter buffer, string text) + where T : struct, IBufferWriter + { + var byteCount = Encoding.UTF8.GetByteCount(text); + buffer.Ensure(byteCount); + byteCount = Encoding.UTF8.GetBytes(text.AsSpan(), buffer.Span); + buffer.Advance(byteCount); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static void WriteNumericMultiWrite(ref this BufferWriter buffer, uint number) + where T : IBufferWriter + { + const byte AsciiDigitStart = (byte)'0'; + + var value = number; + var position = MaxULongByteLength; + var byteBuffer = NumericBytesScratch; + do + { + var quotient = value / 10; + byteBuffer[--position] = (byte)(AsciiDigitStart + (value - quotient * 10)); + value = quotient; + } + while (value != 0); + + var length = MaxULongByteLength - position; + buffer.Write(new ReadOnlySpan(byteBuffer, position, length)); + } + + private static byte[] NumericBytesScratch => s_numericBytesScratch ?? CreateNumericBytesScratch(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static byte[] CreateNumericBytesScratch() + { + var bytes = new byte[MaxULongByteLength]; + s_numericBytesScratch = bytes; + return bytes; + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/BufferWriter.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/BufferWriter.cs new file mode 100644 index 0000000..18d2ca8 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/BufferWriter.cs @@ -0,0 +1,133 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace Slon.Fortunes.Platform; + +public ref struct BufferWriter + where T : IBufferWriter +{ + private T _output; + private Span _span; + private int _buffered; + + public BufferWriter(T output, int sizeHint) + { + _buffered = 0; + _output = output; + _span = output.GetSpan(sizeHint); + } + + public Span Span => _span; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Commit() + { + var buffered = _buffered; + if (buffered > 0) + { + _buffered = 0; + _output.Advance(buffered); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Advance(int count) + { + _buffered += count; + _span = _span[count..]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Write(scoped ReadOnlySpan source) + { + if (_span.Length >= source.Length) + { + source.CopyTo(_span); + Advance(source.Length); + } + else + { + WriteMultiBuffer(source); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Ensure(int count = 1) + { + if (_span.Length < count) + { + EnsureMore(count); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EnsureMore(int count = 0) + { + if (_buffered > 0) + { + Commit(); + } + + _span = _output.GetSpan(count); + } + + private void WriteMultiBuffer(scoped ReadOnlySpan source) + { + while (source.Length > 0) + { + if (_span.Length == 0) + { + EnsureMore(); + } + + var writable = Math.Min(source.Length, _span.Length); + source[..writable].CopyTo(_span); + source = source[writable..]; + Advance(writable); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void WriteNumeric(uint number) + { + const byte AsciiDigitStart = (byte)'0'; + var span = Span; + var advanceBy = 0; + if (span.Length >= 3) + { + if (number < 10) + { + span[0] = (byte)(number + AsciiDigitStart); + advanceBy = 1; + } + else if (number < 100) + { + var tens = (byte)((number * 205u) >> 11); + span[0] = (byte)(tens + AsciiDigitStart); + span[1] = (byte)(number - (tens * 10) + AsciiDigitStart); + advanceBy = 2; + } + else if (number < 1000) + { + var digit0 = (byte)((number * 41u) >> 12); + var digits01 = (byte)((number * 205u) >> 11); + span[0] = (byte)(digit0 + AsciiDigitStart); + span[1] = (byte)(digits01 - (digit0 * 10) + AsciiDigitStart); + span[2] = (byte)(number - (digits01 * 10) + AsciiDigitStart); + advanceBy = 3; + } + } + + if (advanceBy > 0) + { + Advance(advanceBy); + } + else + { + BufferExtensions.WriteNumericMultiWrite(ref this, number); + } + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/ChunkedPipeWriter.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/ChunkedPipeWriter.cs new file mode 100644 index 0000000..411c849 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/ChunkedPipeWriter.cs @@ -0,0 +1,235 @@ +using System.Buffers; +using System.Buffers.Text; +using System.Diagnostics; +using System.IO.Pipelines; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace Slon.Fortunes.Platform; + +internal sealed class ChunkedPipeWriter : PipeWriter +{ + private const int DefaultChunkSizeHint = 2048; + private static readonly StandardFormat DefaultHexFormat = GetHexFormat(DefaultChunkSizeHint); + private static ReadOnlySpan ChunkTerminator => "\r\n"u8; + + private PipeWriter _output = null!; + private int _chunkSizeHint; + private StandardFormat _hexFormat = DefaultHexFormat; + private Memory _currentFullChunk; + private Memory _currentChunk; + private int _buffered; + private long _unflushedBytes; + private bool _ended; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetOutput(PipeWriter output, int chunkSizeHint = DefaultChunkSizeHint) + { + _buffered = 0; + _unflushedBytes = 0; + _chunkSizeHint = chunkSizeHint; + _output = output; + StartNewChunk(chunkSizeHint, isFirst: true); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Reset() + { + _buffered = 0; + _unflushedBytes = 0; + _output = null!; + _ended = false; + _hexFormat = DefaultHexFormat; + _currentFullChunk = default; + _currentChunk = default; + } + + public override bool CanGetUnflushedBytes => true; + + public override long UnflushedBytes => _unflushedBytes; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Advance(int count) + { + ThrowIfEnded(); + _buffered += count; + _unflushedBytes += count; + _currentChunk = _currentChunk[count..]; + } + + public override Memory GetMemory(int sizeHint = 0) + { + ThrowIfEnded(); + if (_currentChunk.Length <= sizeHint) + { + EnsureMore(sizeHint); + } + + return _currentChunk; + } + + public override Span GetSpan(int sizeHint = 0) => GetMemory(sizeHint).Span; + + public override void CancelPendingFlush() => _output.CancelPendingFlush(); + + public override void Complete(Exception? exception = null) + { + ThrowIfEnded(); + CommitCurrentChunk(isFinal: true); + _ended = true; + } + + public override ValueTask FlushAsync(CancellationToken cancellationToken = default) + { + CommitCurrentChunk(isFinal: false); + var flushTask = _output.FlushAsync(cancellationToken); + _unflushedBytes = 0; + return flushTask; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static StandardFormat GetHexFormat(int maxValue) + { + var hexDigitCount = CountHexDigits(maxValue); + return new StandardFormat('X', (byte)hexDigitCount); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int CountHexDigits(int n) => + n <= 16 ? 1 : (BitOperations.Log2((uint)n) >> 2) + 1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void StartNewChunk(int sizeHint, bool isFirst = false) + { + ThrowIfEnded(); + + var oldFullChunkHexLength = -1; + if (!isFirst) + { + oldFullChunkHexLength = CountHexDigits(_currentFullChunk.Length); + } + + _currentFullChunk = _output.GetMemory(Math.Max(_chunkSizeHint, sizeHint)); + var newFullChunkHexLength = CountHexDigits(_currentFullChunk.Length); + var currentFullChunkSpan = _currentFullChunk.Span; + currentFullChunkSpan[..newFullChunkHexLength].Fill((byte)'0'); + "\r\n"u8.CopyTo(currentFullChunkSpan[newFullChunkHexLength..]); + var chunkHeaderLength = newFullChunkHexLength + 2; + _currentChunk = _currentFullChunk[chunkHeaderLength..]; + + if ((!isFirst && oldFullChunkHexLength != newFullChunkHexLength) || + (isFirst && DefaultChunkSizeHint != _chunkSizeHint)) + { + _hexFormat = GetHexFormat(_currentFullChunk.Length); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void CommitCurrentChunk(bool isFinal = false, int sizeHint = 0) + { + ThrowIfEnded(); + + var contentLength = _buffered; + if (contentLength <= 0) + { + if (isFinal) + { + var terminator = "0\r\n\r\n"u8; + terminator.CopyTo(_currentFullChunk.Span); + _output.Advance(terminator.Length); + } + + return; + } + + var chunkLengthHexDigitsLength = CountHexDigits(contentLength); + var span = _currentFullChunk.Span; + if (!Utf8Formatter.TryFormat(contentLength, span, out var bytesWritten, _hexFormat)) + { + throw new NotSupportedException("Chunk size too large"); + } + + Debug.Assert(chunkLengthHexDigitsLength == bytesWritten, "HEX formatting math problem."); + var spanOffset = chunkLengthHexDigitsLength + 2 + contentLength; + var chunkTotalLength = spanOffset + ChunkTerminator.Length; + Debug.Assert(span.Length >= chunkTotalLength, "Bad chunk size calculation."); + ChunkTerminator.CopyTo(span[spanOffset..]); + + if (!isFinal) + { + _output.Advance(chunkTotalLength); + StartNewChunk(sizeHint); + } + else + { + var terminator = "0\r\n\r\n"u8; + if (chunkTotalLength + terminator.Length <= span.Length) + { + terminator.CopyTo(span[chunkTotalLength..]); + _output.Advance(chunkTotalLength + terminator.Length); + } + else + { + _output.Advance(chunkTotalLength); + _output.Write(terminator); + } + } + + _buffered = 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Write(ReadOnlySpan source) + { + ThrowIfEnded(); + if (_currentChunk.Length >= source.Length + ChunkTerminator.Length) + { + source.CopyTo(_currentChunk.Span); + Advance(source.Length); + } + else + { + WriteMultiBuffer(source); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void EnsureMore(int count = 0) + { + if (count > _currentChunk.Length - _buffered - ChunkTerminator.Length) + { + if (_buffered > 0) + { + CommitCurrentChunk(isFinal: false, count); + } + else + { + StartNewChunk(count); + } + } + } + + private void WriteMultiBuffer(ReadOnlySpan source) + { + while (source.Length > 0) + { + if (_currentChunk.Length - ChunkTerminator.Length == 0) + { + EnsureMore(); + } + + var writable = Math.Min(source.Length, _currentChunk.Length - ChunkTerminator.Length); + source[..writable].CopyTo(_currentChunk.Span); + source = source[writable..]; + Advance(writable); + } + } + + private void ThrowIfEnded() + { + if (_ended) + { + throw new InvalidOperationException("Cannot use the writer after calling End()."); + } + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/DateHeader.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/DateHeader.cs new file mode 100644 index 0000000..0782086 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/DateHeader.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +using System.Buffers.Text; +using System.Diagnostics; + +namespace Slon.Fortunes.Platform; + +internal static class DateHeader +{ + private const int PrefixLength = 8; + private const int DateTimeRLength = 29; + private const int SuffixLength = 2; + private const int SuffixIndex = DateTimeRLength + PrefixLength; + + private static readonly Timer s_timer = new(static _ => SetDateValues(DateTimeOffset.UtcNow), null, 1000, 1000); + private static byte[] s_headerBytesMaster = new byte[PrefixLength + DateTimeRLength + 2 * SuffixLength]; + private static byte[] s_headerBytesScratch = new byte[PrefixLength + DateTimeRLength + 2 * SuffixLength]; + + static DateHeader() + { + "\r\nDate: "u8.CopyTo(s_headerBytesMaster); + "\r\nDate: "u8.CopyTo(s_headerBytesScratch); + s_headerBytesMaster[SuffixIndex] = (byte)'\r'; + s_headerBytesMaster[SuffixIndex + 1] = (byte)'\n'; + s_headerBytesMaster[SuffixIndex + 2] = (byte)'\r'; + s_headerBytesMaster[SuffixIndex + 3] = (byte)'\n'; + s_headerBytesScratch[SuffixIndex] = (byte)'\r'; + s_headerBytesScratch[SuffixIndex + 1] = (byte)'\n'; + s_headerBytesScratch[SuffixIndex + 2] = (byte)'\r'; + s_headerBytesScratch[SuffixIndex + 3] = (byte)'\n'; + SetDateValues(DateTimeOffset.UtcNow); + SyncDateTimer(); + } + + public static void SyncDateTimer() => s_timer.Change(1000, 1000); + + public static ReadOnlySpan HeaderBytes => s_headerBytesMaster; + + private static void SetDateValues(DateTimeOffset value) + { + lock (s_headerBytesScratch) + { + if (!Utf8Formatter.TryFormat(value, s_headerBytesScratch.AsSpan(PrefixLength), out var written, 'R')) + { + throw new Exception("date time format failed"); + } + + Debug.Assert(written == DateTimeRLength); + (s_headerBytesScratch, s_headerBytesMaster) = (s_headerBytesMaster, s_headerBytesScratch); + } + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs new file mode 100644 index 0000000..2c26aef --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs @@ -0,0 +1,17 @@ +namespace Slon.Fortunes.Platform; + +public readonly struct Fortune : IComparable +{ + public Fortune(int id, string message) + { + Id = id; + Message = message; + } + + public int Id { get; } + + public string Message { get; } + + public int CompareTo(Fortune other) => + StringComparer.Ordinal.Compare(Message, other.Message); +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs new file mode 100644 index 0000000..5e886d4 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs @@ -0,0 +1,189 @@ +using System.Globalization; +using System.Net; +using Npgsql; + +namespace Slon.Fortunes.Platform; + +internal abstract class FortuneDatabase : IAsyncDisposable +{ + internal const string Query = "SELECT id, message FROM fortune"; + private const string AdditionalFortune = "Additional fortune added at request time."; + + public abstract ValueTask DisposeAsync(); + + public abstract ValueTask> LoadAsync( + CancellationToken cancellationToken); + + public static ValueTask CreateAsync( + string? database, + string? driver, + string? connectionString) + { + var selectedDatabase = RequiredSelection("DATABASE", database); + var selectedDriver = RequiredSelection("DRIVER", driver); + var requiredConnectionString = string.IsNullOrWhiteSpace(connectionString) + ? throw new InvalidOperationException("CONNECTION_STRING is required.") + : connectionString; + var connectionCount = PositiveEnvironment("DATABASE_CONNECTIONS"); + + return (selectedDatabase, selectedDriver) switch + { + ("postgresql", "slon") => SlonFortuneDatabase.CreateAsync( + requiredConnectionString, + connectionCount, + PositiveEnvironment("SLON_PIPELINING")), + ("postgresql", "npgsql") => + ValueTask.FromResult( + new NpgsqlFortuneDatabase(requiredConnectionString, connectionCount)), + ("postgresql", _) => + throw new InvalidOperationException( + $"DRIVER '{selectedDriver}' is not valid for DATABASE '{selectedDatabase}'."), + _ => throw new ArgumentOutOfRangeException( + nameof(database), + selectedDatabase, + "DATABASE must be 'postgresql'."), + }; + } + + protected static List Complete(List fortunes) + { + fortunes.Add(new Fortune(0, AdditionalFortune)); + fortunes.Sort(); + return fortunes; + } + + protected static int PositiveEnvironment(string name) + { + var value = Environment.GetEnvironmentVariable(name) ?? + throw new InvalidOperationException($"{name} is required."); + + return int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) && + parsed > 0 + ? parsed + : throw new ArgumentOutOfRangeException( + name, + value, + "Value must be a positive integer."); + } + + private static string RequiredSelection(string name, string? value) => + string.IsNullOrWhiteSpace(value) + ? throw new InvalidOperationException($"{name} is required.") + : value.Trim().ToLowerInvariant(); +} + +internal sealed class SlonFortuneDatabase : FortuneDatabase +{ + private readonly SlonDataSource _dataSource; + private readonly SlonCommand _command; + + private SlonFortuneDatabase(SlonDataSource dataSource, SlonCommand command) + { + _dataSource = dataSource; + _command = command; + } + + public static async ValueTask CreateAsync( + string connectionString, + int connectionCount, + int pipeliningLimit) + { + var builder = new NpgsqlConnectionStringBuilder(connectionString); + var dataSource = new SlonDataSource(new SlonDataSourceOptions + { + EndPoint = new DnsEndPoint( + RequiredPostgreSqlValue("Host", builder.Host), + builder.Port), + Database = RequiredPostgreSqlValue("Database", builder.Database), + Username = RequiredPostgreSqlValue("Username", builder.Username), + Password = builder.Password, + PoolSize = connectionCount, + MaxInFlightOperationsPerWire = pipeliningLimit, + Ssl = new PostgreSqlSslOptions + { + Mode = PostgreSqlSslMode.Disable, + }, + }); + + try + { + var command = dataSource.CreateCommand(Query); + try + { + await command.PrepareAsync(); + return new SlonFortuneDatabase(dataSource, command); + } + catch + { + await command.DisposeAsync(); + throw; + } + } + catch + { + await dataSource.DisposeAsync(); + throw; + } + } + + public override async ValueTask> LoadAsync( + CancellationToken cancellationToken) + { + await using var reader = await _command.ExecuteReaderAsync(cancellationToken); + List fortunes = []; + while (await reader.ReadAsync(cancellationToken)) + { + fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); + } + + return Complete(fortunes); + } + + public override async ValueTask DisposeAsync() + { + try + { + await _command.DisposeAsync(); + } + finally + { + await _dataSource.DisposeAsync(); + } + } + + private static string RequiredPostgreSqlValue(string name, string? value) => + string.IsNullOrWhiteSpace(value) + ? throw new InvalidOperationException($"PostgreSQL {name} is required.") + : value; +} + +internal sealed class NpgsqlFortuneDatabase : FortuneDatabase +{ + private readonly NpgsqlDataSource _dataSource; + + public NpgsqlFortuneDatabase(string connectionString, int connectionCount) + { + var builder = new NpgsqlConnectionStringBuilder(connectionString) + { + MaxPoolSize = connectionCount, + }; + _dataSource = new NpgsqlSlimDataSourceBuilder(builder.ConnectionString).Build(); + } + + public override async ValueTask> LoadAsync( + CancellationToken cancellationToken) + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var command = new NpgsqlCommand(Query, connection); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + List fortunes = []; + while (await reader.ReadAsync(cancellationToken)) + { + fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); + } + + return Complete(fortunes); + } + + public override ValueTask DisposeAsync() => _dataSource.DisposeAsync(); +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/HttpApplication.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/HttpApplication.cs new file mode 100644 index 0000000..54666d4 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/HttpApplication.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +using Microsoft.AspNetCore.Connections; + +namespace Slon.Fortunes.Platform; + +public static class HttpApplicationConnectionBuilderExtensions +{ + public static IConnectionBuilder UseHttpApplication(this IConnectionBuilder builder) + where TConnection : IHttpConnection, new() => + builder.Use(_ => new HttpApplication().ExecuteAsync); +} + +public sealed class HttpApplication + where TConnection : IHttpConnection, new() +{ + public Task ExecuteAsync(ConnectionContext connection) + { + var httpConnection = new TConnection + { + ConnectionClosed = connection.ConnectionClosed, + Reader = connection.Transport.Input, + Writer = connection.Transport.Output, + }; + return httpConnection.ExecuteAsync(); + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/IHttpConnection.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/IHttpConnection.cs new file mode 100644 index 0000000..ce144cd --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/IHttpConnection.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +using System.IO.Pipelines; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; + +namespace Slon.Fortunes.Platform; + +public interface IHttpConnection : IHttpHeadersHandler, IHttpRequestLineHandler +{ + CancellationToken ConnectionClosed { get; set; } + + PipeReader Reader { get; set; } + + PipeWriter Writer { get; set; } + + Task ExecuteAsync(); +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs new file mode 100644 index 0000000..d611964 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs @@ -0,0 +1,49 @@ +using System.Net; +using System.Runtime.InteropServices; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Slon.Fortunes.Platform; + +var configuration = new ConfigurationBuilder() + .AddEnvironmentVariables() + .AddCommandLine(args) + .Build(); +var url = new Uri(configuration["urls"] ?? "http://0.0.0.0:5000"); + +await using var database = await FortuneDatabase.CreateAsync( + configuration["DATABASE"], + configuration["DRIVER"], + configuration["CONNECTION_STRING"]); +BenchmarkApplication.Database = database; +DateHeader.SyncDateTimer(); + +var hostBuilder = Host.CreateDefaultBuilder() + .ConfigureWebHost(webHost => + { + webHost + .UseConfiguration(configuration) + .UseKestrel(options => + { + options.Listen(IPAddress.Any, url.Port, listen => + { + listen.UseHttpApplication(); + }); + }) + .Configure(_ => { }) + .UseSockets(options => + { + options.WaitForDataBeforeAllocatingBuffer = false; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + options.UnsafePreferInlineScheduling = + Environment.GetEnvironmentVariable( + "DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS") == "1"; + } + }); + }); + +using var host = hostBuilder.Build(); +await host.StartAsync(); +Console.WriteLine("Application started."); +await host.WaitForShutdownAsync(); diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md new file mode 100644 index 0000000..fc608de --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -0,0 +1,32 @@ +# Slon Platform Fortunes + +This standalone Platform-style Fortunes application exposes `GET /fortunes`. It reads every +row from `fortune`, adds the standard request-time fortune, sorts by message, and renders the +standard HTML response with RazorSlices HTML encoding. + +## Selection + +Set all of these environment variables before starting the app: + +| Variable | Values | +| --- | --- | +| `DATABASE` | `postgresql` | +| `DRIVER` | `slon` or `npgsql` | +| `CONNECTION_STRING` | PostgreSQL connection string | +| `DATABASE_CONNECTIONS` | Positive fixed pool size | +| `SLON_PIPELINING` | Positive per-wire in-flight limit for Slon | + +Invalid, unsupported, or missing selections fail application startup with an explicit error. +The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an +unmerged branch. + +## Driver strategies + +Slon uses a fixed-size `SlonDataSource` and one data-source-bound command prepared at startup. +The prepared command is reused concurrently and pipelines requests across the configured +connections. Npgsql uses a slim data source and a command bound to each leased connection. +Both drivers materialize messages with `GetString`, append and ordinally sort the same string +model, and render the same RazorSlices string template for a fair comparison. + +The Crank configuration uses two fewer Slon connections than database cores and 256 Npgsql +connections. `SLON_PIPELINING` is set to 64 for the low-level Platform benchmark. diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj new file mode 100644 index 0000000..6bb484f --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj @@ -0,0 +1,20 @@ + + + + net10.0 + enable + enable + false + $(DefineConstants);DATABASE + + + + + + + + + + + + diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml new file mode 100644 index 0000000..39a4cb9 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml @@ -0,0 +1,2 @@ +@inherits RazorSlice> +Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message
diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/_ViewImports.cshtml b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/_ViewImports.cshtml new file mode 100644 index 0000000..068bf13 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/_ViewImports.cshtml @@ -0,0 +1,9 @@ +@inherits RazorSlice + +@using System.Globalization +@using Microsoft.AspNetCore.Razor +@using RazorSlices +@using Slon.Fortunes.Platform + +@tagHelperPrefix __disable_tagHelpers__: +@removeTagHelper *, Microsoft.AspNetCore.Mvc.Razor diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml b/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml new file mode 100644 index 0000000..9147b29 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml @@ -0,0 +1,88 @@ +imports: + - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml + - https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/aspnet.profiles.standard.yml + +variables: + serverPort: 5000 + npgsqlConnections: 256 + # Override this value to benchmark an unmerged Slon branch or commit. + branchOrCommit: main + +jobs: + platform-postgresql-slon: + sources: + Slon: + repository: https://github.com/draghidev/slon.git + branchOrCommit: "{{branchOrCommit}}" + Draghi: + repository: https://github.com/draghidev/pipelining.git + branchOrCommit: main + project: Slon/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj + framework: net10.0 + patchReferences: true + readyStateText: Application started. + arguments: "--urls {{serverScheme}}://{{serverAddress}}:{{serverPort}}" + variables: + serverScheme: http + environmentVariables: + DATABASE: postgresql + DRIVER: slon + CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass" + DATABASE_CONNECTIONS: "{{ cores | minus: 2 }}" + SLON_PIPELINING: 64 + + platform-postgresql-npgsql: + sources: + Slon: + repository: https://github.com/draghidev/slon.git + branchOrCommit: "{{branchOrCommit}}" + Draghi: + repository: https://github.com/draghidev/pipelining.git + branchOrCommit: main + project: Slon/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj + framework: net10.0 + patchReferences: true + readyStateText: Application started. + arguments: "--urls {{serverScheme}}://{{serverAddress}}:{{serverPort}}" + variables: + serverScheme: http + environmentVariables: + DATABASE: postgresql + DRIVER: npgsql + CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass;Maximum Pool Size={{ npgsqlConnections }};No Reset On Close=true;Enlist=false;Max Auto Prepare=4" + DATABASE_CONNECTIONS: "{{ npgsqlConnections }}" + + postgresql: + source: + repository: https://github.com/TechEmpower/FrameworkBenchmarks.git + branchOrCommit: master + dockerFile: toolset/databases/postgres/postgres.dockerfile + dockerImageName: postgres_te + dockerContextDirectory: toolset/databases/postgres + readyStateText: ready to accept connections + noClean: true + +scenarios: + platform-postgresql-slon: + db: + job: postgresql + application: + job: platform-postgresql-slon + load: + job: wrk + variables: + presetHeaders: html + path: /fortunes + connections: 512 + + platform-postgresql-npgsql: + db: + job: postgresql + application: + job: platform-postgresql-npgsql + load: + job: wrk + variables: + presetHeaders: html + path: /fortunes + connections: 512 diff --git a/Slon.slnx b/Slon.slnx index f4c0908..56b8f3d 100644 --- a/Slon.slnx +++ b/Slon.slnx @@ -3,5 +3,7 @@ + + From 04016c690db4e48dc5008a5c417294a950c81b1b Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 28 Aug 2026 00:20:19 +0200 Subject: [PATCH 126/136] Benchmark Slon through its lower protocol layer Use an atomic round-robin protocol pool so the Fortunes scenarios measure Slon command flows independently of the production datasource pool. --- Slon.Benchmarks/Shared/RawSlonProtocolPool.cs | 130 ++++++++++++++++++ .../Slon.Fortunes.Minimal/FortuneDatabase.cs | 81 ++--------- .../Slon.Fortunes.Minimal/README.md | 17 ++- .../Slon.Fortunes.Minimal.csproj | 2 + .../minimal-fortunes.benchmarks.yml | 1 - .../Slon.Fortunes.Platform/FortuneDatabase.cs | 81 ++--------- .../Slon.Fortunes.Platform/README.md | 18 ++- .../Slon.Fortunes.Platform.csproj | 2 + .../platform-fortunes.benchmarks.yml | 1 - 9 files changed, 175 insertions(+), 158 deletions(-) create mode 100644 Slon.Benchmarks/Shared/RawSlonProtocolPool.cs diff --git a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs new file mode 100644 index 0000000..b594ba6 --- /dev/null +++ b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs @@ -0,0 +1,130 @@ +using System.Net; +using Npgsql; +using Slon.Pg; +using Slon.Pg.Protocol; +using Slon.Pg.Protocol.Flows; +using Slon.Text; +using Slon.Transport; + +namespace Slon.Fortunes; + +internal sealed class RawSlonProtocolPool : IAsyncDisposable +{ + const string Query = "SELECT id, message FROM fortune"; + readonly Slot[] _slots; + int _nextSlot = -1; + + RawSlonProtocolPool(Slot[] slots) => _slots = slots; + + internal static async ValueTask CreateAsync( + string connectionString, + int connectionCount) + { + var builder = new NpgsqlConnectionStringBuilder(connectionString); + var clientOptions = new PgClientOptions + { + EndPoint = new DnsEndPoint( + RequiredPostgreSqlValue("Host", builder.Host), + builder.Port), + Database = RequiredPostgreSqlValue("Database", builder.Database), + Username = RequiredPostgreSqlValue("Username", builder.Username), + Password = builder.Password, + Ssl = new PostgreSqlSslOptions { Mode = PostgreSqlSslMode.Disable }, + }; + var factory = new PgClientProtocolFactory( + clientOptions, + SocketStreamConnection.CreateFactory(clientOptions.EndPoint, new TransportConnectionOptions + { + // Match Apex's ordinary BCL read shape for this lower-layer ceiling comparison. + UseZeroByteReads = false, + })); + var slots = new Slot[connectionCount]; + var created = 0; + try + { + for (; created < slots.Length; created++) + { + var protocol = await factory.CreateAsync().ConfigureAwait(false); + try + { + slots[created] = new(protocol, await PrepareAsync(protocol).ConfigureAwait(false)); + } + catch + { + await protocol.DisposeAsync().ConfigureAwait(false); + throw; + } + } + return new(slots); + } + catch + { + for (var i = 0; i < created; i++) + await slots[i].Protocol.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + internal async ValueTask> LoadAsync( + Func create, + CancellationToken cancellationToken) + { + var slot = GetSlot(); + var flow = new ReaderDrivenCommandFlow(slot.Command); + if (!slot.Protocol.TryQueue(flow, cancellationToken: cancellationToken)) + throw new InvalidOperationException("The selected PostgreSQL protocol is unavailable."); + + List values = []; + await foreach (var result in flow) + { + await foreach (var row in result) + values.Add(create(row.GetValue(0), row.GetValue(1))); + } + return values; + } + + Slot GetSlot() + => _slots[(int)((uint)Interlocked.Increment(ref _nextSlot) % (uint)_slots.Length)]; + + public async ValueTask DisposeAsync() + { + List? errors = null; + foreach (var slot in _slots) + { + try + { + await slot.Protocol.DisposeAsync().ConfigureAwait(false); + } + catch (Exception exception) + { + (errors ??= []).Add(exception); + } + } + if (errors is not null) + throw errors.Count is 1 ? errors[0] : new AggregateException(errors); + } + + static async ValueTask PrepareAsync(PgClientProtocol protocol) + { + var command = Command.Create(Query, commandName: new EncodedCString("fortunes")) with + { + DescribeOnly = true, + DescribeForPreparation = true, + }; + var flow = protocol.Queue(new CommandFlow(async: true, command)); + await foreach (var result in flow) + return Command.Create(result.GetMetadata().ToPreparedDescriptor()); + throw new InvalidOperationException("PostgreSQL preparation returned no command result."); + } + + static string RequiredPostgreSqlValue(string name, string? value) => + string.IsNullOrWhiteSpace(value) + ? throw new InvalidOperationException($"PostgreSQL {name} is required.") + : value; + + sealed class Slot(PgClientProtocol protocol, Command command) + { + internal PgClientProtocol Protocol { get; } = protocol; + internal Command Command { get; } = command; + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs index 73f91cd..70b75a0 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs @@ -1,6 +1,6 @@ using System.Globalization; -using System.Net; using Npgsql; +using Slon.Fortunes; namespace Slon.Fortunes.Minimal; @@ -27,8 +27,7 @@ public static ValueTask CreateAsync(IConfiguration configuratio { ("postgresql", "slon") => SlonFortuneDatabase.CreateAsync( connectionString, - connectionCount, - PositiveSetting(configuration, "SLON_PIPELINING")), + connectionCount), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(connectionString, connectionCount)), _ => throw new InvalidOperationException("The database selection is invalid."), @@ -82,87 +81,27 @@ private static int PositiveSetting(IConfiguration configuration, string name) internal sealed class SlonFortuneDatabase : FortuneDatabase { - private readonly SlonDataSource _dataSource; - private readonly SlonCommand _command; + private readonly RawSlonProtocolPool _pool; - private SlonFortuneDatabase(SlonDataSource dataSource, SlonCommand command) - { - _dataSource = dataSource; - _command = command; - } + private SlonFortuneDatabase(RawSlonProtocolPool pool) => _pool = pool; public static async ValueTask CreateAsync( string connectionString, - int connectionCount, - int pipeliningLimit) + int connectionCount) { - var builder = new NpgsqlConnectionStringBuilder(connectionString); - var dataSource = new SlonDataSource(new SlonDataSourceOptions - { - EndPoint = new DnsEndPoint( - RequiredPostgreSqlValue("Host", builder.Host), - builder.Port), - Database = RequiredPostgreSqlValue("Database", builder.Database), - Username = RequiredPostgreSqlValue("Username", builder.Username), - Password = builder.Password, - PoolSize = connectionCount, - MaxInFlightOperationsPerWire = pipeliningLimit, - Ssl = new PostgreSqlSslOptions - { - Mode = PostgreSqlSslMode.Disable, - }, - }); - - try - { - var command = dataSource.CreateCommand(Query); - try - { - await command.PrepareAsync(); - return new SlonFortuneDatabase(dataSource, command); - } - catch - { - await command.DisposeAsync(); - throw; - } - } - catch - { - await dataSource.DisposeAsync(); - throw; - } + return new SlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( + connectionString, connectionCount).ConfigureAwait(false)); } public override async ValueTask> LoadAsync( CancellationToken cancellationToken) { - await using var reader = await _command.ExecuteReaderAsync(cancellationToken); - List fortunes = []; - while (await reader.ReadAsync(cancellationToken)) - { - fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); - } - + var fortunes = await _pool.LoadAsync( + static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); return Complete(fortunes); } - public override async ValueTask DisposeAsync() - { - try - { - await _command.DisposeAsync(); - } - finally - { - await _dataSource.DisposeAsync(); - } - } - - private static string RequiredPostgreSqlValue(string name, string? value) => - string.IsNullOrWhiteSpace(value) - ? throw new InvalidOperationException($"PostgreSQL {name} is required.") - : value; + public override ValueTask DisposeAsync() => _pool.DisposeAsync(); } internal sealed class NpgsqlFortuneDatabase : FortuneDatabase diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md index 2efedde..11f7c5e 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -14,7 +14,6 @@ Set the following configuration values as environment variables or equivalent .N | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | -| `SLON_PIPELINING` | Positive per-wire in-flight limit for Slon | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -22,12 +21,16 @@ unmerged branch. ## Driver strategies -Slon uses a fixed-size `SlonDataSource` and one data-source-bound command prepared at startup. -The prepared command is reused concurrently and pipelines requests across the configured -connections. Npgsql uses a slim data source and a command bound to each leased connection. -Both drivers materialize messages with `GetString`, append and ordinally sort the same string -model, and render the same RazorSlices string template for a fair comparison. +Slon uses its experimental lower layer directly. A benchmark-local fixed pool opens +`DATABASE_CONNECTIONS` `PgClientProtocol` instances, prepares the statement once on every wire, +and places flows by atomic round-robin. This deliberately simple outer pool isolates Slon's +protocol/flow baseline. It does not exercise the richer production `SlonDataSource` placement +policy. +The raw Slon arm disables zero-byte reads to match Apex's ordinary BCL transport shape. + +Npgsql uses a slim data source and a command bound to each leased connection. Both drivers +materialize messages as strings, append and ordinally sort the same model, and render the same +RazorSlices string template for a fair comparison. The Crank configuration uses two fewer Slon connections than database cores and 256 Npgsql connections; Npgsql needs the additional in-flight operations to hide network and query latency. -`SLON_PIPELINING` is set to 16. diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj index e6ff620..019cd04 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj @@ -5,10 +5,12 @@ enable enable false + $(NoWarn);SLONPG001 + diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml b/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml index 88a7858..80cca1a 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml @@ -29,7 +29,6 @@ jobs: DRIVER: slon CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass" DATABASE_CONNECTIONS: "{{ cores | minus: 2 }}" - SLON_PIPELINING: 16 minimal-postgresql-npgsql: sources: diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs index 5e886d4..5231a88 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs @@ -1,6 +1,6 @@ using System.Globalization; -using System.Net; using Npgsql; +using Slon.Fortunes; namespace Slon.Fortunes.Platform; @@ -30,8 +30,7 @@ public static ValueTask CreateAsync( { ("postgresql", "slon") => SlonFortuneDatabase.CreateAsync( requiredConnectionString, - connectionCount, - PositiveEnvironment("SLON_PIPELINING")), + connectionCount), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(requiredConnectionString, connectionCount)), @@ -74,87 +73,27 @@ private static string RequiredSelection(string name, string? value) => internal sealed class SlonFortuneDatabase : FortuneDatabase { - private readonly SlonDataSource _dataSource; - private readonly SlonCommand _command; + private readonly RawSlonProtocolPool _pool; - private SlonFortuneDatabase(SlonDataSource dataSource, SlonCommand command) - { - _dataSource = dataSource; - _command = command; - } + private SlonFortuneDatabase(RawSlonProtocolPool pool) => _pool = pool; public static async ValueTask CreateAsync( string connectionString, - int connectionCount, - int pipeliningLimit) + int connectionCount) { - var builder = new NpgsqlConnectionStringBuilder(connectionString); - var dataSource = new SlonDataSource(new SlonDataSourceOptions - { - EndPoint = new DnsEndPoint( - RequiredPostgreSqlValue("Host", builder.Host), - builder.Port), - Database = RequiredPostgreSqlValue("Database", builder.Database), - Username = RequiredPostgreSqlValue("Username", builder.Username), - Password = builder.Password, - PoolSize = connectionCount, - MaxInFlightOperationsPerWire = pipeliningLimit, - Ssl = new PostgreSqlSslOptions - { - Mode = PostgreSqlSslMode.Disable, - }, - }); - - try - { - var command = dataSource.CreateCommand(Query); - try - { - await command.PrepareAsync(); - return new SlonFortuneDatabase(dataSource, command); - } - catch - { - await command.DisposeAsync(); - throw; - } - } - catch - { - await dataSource.DisposeAsync(); - throw; - } + return new SlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( + connectionString, connectionCount).ConfigureAwait(false)); } public override async ValueTask> LoadAsync( CancellationToken cancellationToken) { - await using var reader = await _command.ExecuteReaderAsync(cancellationToken); - List fortunes = []; - while (await reader.ReadAsync(cancellationToken)) - { - fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); - } - + var fortunes = await _pool.LoadAsync( + static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); return Complete(fortunes); } - public override async ValueTask DisposeAsync() - { - try - { - await _command.DisposeAsync(); - } - finally - { - await _dataSource.DisposeAsync(); - } - } - - private static string RequiredPostgreSqlValue(string name, string? value) => - string.IsNullOrWhiteSpace(value) - ? throw new InvalidOperationException($"PostgreSQL {name} is required.") - : value; + public override ValueTask DisposeAsync() => _pool.DisposeAsync(); } internal sealed class NpgsqlFortuneDatabase : FortuneDatabase diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index fc608de..a477bd2 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -14,7 +14,6 @@ Set all of these environment variables before starting the app: | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | -| `SLON_PIPELINING` | Positive per-wire in-flight limit for Slon | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -22,11 +21,16 @@ unmerged branch. ## Driver strategies -Slon uses a fixed-size `SlonDataSource` and one data-source-bound command prepared at startup. -The prepared command is reused concurrently and pipelines requests across the configured -connections. Npgsql uses a slim data source and a command bound to each leased connection. -Both drivers materialize messages with `GetString`, append and ordinally sort the same string -model, and render the same RazorSlices string template for a fair comparison. +Slon uses its experimental lower layer directly. A benchmark-local fixed pool opens +`DATABASE_CONNECTIONS` `PgClientProtocol` instances, prepares the statement once on every wire, +and places flows by atomic round-robin. This deliberately simple outer pool isolates Slon's +protocol/flow baseline. It does not exercise the richer production `SlonDataSource` placement +policy. +The raw Slon arm disables zero-byte reads to match Apex's ordinary BCL transport shape. + +Npgsql uses a slim data source and a command bound to each leased connection. Both drivers +materialize messages as strings, append and ordinally sort the same model, and render the same +RazorSlices string template for a fair comparison. The Crank configuration uses two fewer Slon connections than database cores and 256 Npgsql -connections. `SLON_PIPELINING` is set to 64 for the low-level Platform benchmark. +connections. diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj index 6bb484f..8cc3a6e 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj @@ -6,10 +6,12 @@ enable false $(DefineConstants);DATABASE + $(NoWarn);SLONPG001 + diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml b/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml index 9147b29..09ce85a 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml @@ -29,7 +29,6 @@ jobs: DRIVER: slon CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass" DATABASE_CONNECTIONS: "{{ cores | minus: 2 }}" - SLON_PIPELINING: 64 platform-postgresql-npgsql: sources: From 60a7f6f4bc7042ecec015299bef3e9b1fc5a22b1 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 28 Aug 2026 14:34:16 +0200 Subject: [PATCH 127/136] Compare raw and production pool placement --- .../Shared/FullSlonConnectionPool.cs | 162 ++++++++++++++++++ Slon.Benchmarks/Shared/RawSlonProtocolPool.cs | 11 +- .../Slon.Fortunes.Minimal/FortuneDatabase.cs | 55 ++++-- .../Slon.Fortunes.Minimal/README.md | 13 +- .../Slon.Fortunes.Minimal.csproj | 3 +- .../Slon.Fortunes.Platform/FortuneDatabase.cs | 55 ++++-- .../Slon.Fortunes.Platform/README.md | 13 +- .../Slon.Fortunes.Platform.csproj | 3 +- 8 files changed, 271 insertions(+), 44 deletions(-) create mode 100644 Slon.Benchmarks/Shared/FullSlonConnectionPool.cs diff --git a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs new file mode 100644 index 0000000..d092c96 --- /dev/null +++ b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs @@ -0,0 +1,162 @@ +using System.Net; +using Npgsql; +using Slon.Pg; +using Slon.Pg.Protocol; +using Slon.Pg.Protocol.Flows; +using Slon.Pooling; +using Slon.Text; +using Slon.Transport; + +namespace Slon.Fortunes; + +internal sealed class FullSlonConnectionPool : IAsyncDisposable +{ + const string Query = "SELECT id, message FROM fortune"; + readonly ConnectionPool _pool; + readonly Command _command; + + FullSlonConnectionPool(ConnectionPool pool, Command command) + => (_pool, _command) = (pool, command); + + internal static async ValueTask CreateAsync( + string connectionString, + int connectionCount) + { + var builder = new NpgsqlConnectionStringBuilder(connectionString); + var clientOptions = new PgClientOptions + { + EndPoint = new DnsEndPoint( + RequiredPostgreSqlValue("Host", builder.Host), builder.Port), + Database = RequiredPostgreSqlValue("Database", builder.Database), + Username = RequiredPostgreSqlValue("Username", builder.Username), + Password = builder.Password, + Ssl = new PostgreSqlSslOptions { Mode = PostgreSqlSslMode.Disable }, + }; + var protocolFactory = new PgClientProtocolFactory( + clientOptions, + SocketStreamConnection.CreateFactory(clientOptions.EndPoint, new TransportConnectionOptions + { + UseZeroByteReads = false, + })); + + // Every pooled wire installs the same named statement. Obtain its immutable descriptor once; + // later flows can be created before placement and use it on whichever wire the pool selects. + Command command; + await using (var bootstrap = await protocolFactory.CreateAsync().ConfigureAwait(false)) + command = await PrepareAsync(bootstrap).ConfigureAwait(false); + + var pool = new ConnectionPool( + new ProtocolConnectionFactory(protocolFactory), + new ConnectionPoolOptions + { + MinConnections = connectionCount, + MaxConnections = connectionCount, + ConnectionIdleLifetime = Timeout.InfiniteTimeSpan, + }); + return new(pool, command); + } + + public async ValueTask> LoadAsync( + Func create, + CancellationToken cancellationToken) + { + var flow = new ReaderDrivenCommandFlow(_command); + await _pool.GetAsync( + static (candidate, item) => candidate.Connection.Protocol.TryQueue( + item, + candidate.IsIdleCandidate + ? FlowEnqueueOptions.None + : FlowEnqueueOptions.RequireExistingPipeline, + candidate.CancellationToken), + flow, + Timeout.InfiniteTimeSpan, + cancellationToken).ConfigureAwait(false); + + List values = []; + await foreach (var result in flow) + await foreach (var row in result) + values.Add(create(row.GetValue(0), row.GetValue(1))); + return values; + } + + public ValueTask DisposeAsync() => _pool.DisposeAsync(); + + static async ValueTask PrepareAsync(PgClientProtocol protocol) + { + var command = Command.Create(Query, commandName: new EncodedCString("fortunes")) with + { + DescribeOnly = true, + DescribeForPreparation = true, + WithSync = true, + }; + var flow = protocol.Queue(new CommandFlow(async: true, command)); + await foreach (var result in flow) + { + var metadata = result.GetMetadata(); + return Command.Create(CommandDescriptor.CreatePrepared( + metadata.CommandName, + metadata.ParameterTypes.Preserve(), + metadata.RowDescription?.Preserve())); + } + throw new InvalidOperationException("PostgreSQL preparation returned no command result."); + } + + static string RequiredPostgreSqlValue(string name, string? value) + => string.IsNullOrWhiteSpace(value) + ? throw new InvalidOperationException($"PostgreSQL {name} is required.") + : value; + + sealed class ProtocolConnection(PgClientProtocol protocol) + : IPoolConnection + { + internal PgClientProtocol Protocol { get; } = protocol; + public bool IsIdle => Protocol.Outstanding == 0; + public bool IsSchedulable => Protocol.IsSchedulable; + public Task Completion => Protocol.Completion; + public Task CompleteAsync(Exception? exception = null) => Protocol.CompleteAsync(exception); + public int CompareTo(ProtocolConnection? other) + => other is null ? 1 : Protocol.Outstanding.CompareTo(other.Protocol.Outstanding); + + public void Start(ConnectionPool.Registration registration) + => Protocol.SetAdmissionAvailableCallback( + () => registration.SignalAvailability(Protocol.Outstanding == 0)); + } + + sealed class ProtocolConnectionFactory(PgClientProtocolFactory factory) + : IPoolConnectionFactory + { + public ProtocolConnection Create( + ConnectionPoolContext poolContext, + TimeSpan timeout = default) + { + var protocol = factory.Create(timeout); + try + { + _ = PrepareAsync(protocol).AsTask().GetAwaiter().GetResult(); + return new(protocol); + } + catch + { + protocol.Dispose(); + throw; + } + } + + public async ValueTask CreateAsync( + ConnectionPoolContext poolContext, + CancellationToken cancellationToken = default) + { + var protocol = await factory.CreateAsync(cancellationToken).ConfigureAwait(false); + try + { + _ = await PrepareAsync(protocol).ConfigureAwait(false); + return new(protocol); + } + catch + { + await protocol.DisposeAsync().ConfigureAwait(false); + throw; + } + } + } +} diff --git a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs index b594ba6..b84a17e 100644 --- a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs +++ b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs @@ -65,7 +65,7 @@ internal static async ValueTask CreateAsync( } } - internal async ValueTask> LoadAsync( + public async ValueTask> LoadAsync( Func create, CancellationToken cancellationToken) { @@ -110,10 +110,17 @@ static async ValueTask PrepareAsync(PgClientProtocol protocol) { DescribeOnly = true, DescribeForPreparation = true, + WithSync = true, }; var flow = protocol.Queue(new CommandFlow(async: true, command)); await foreach (var result in flow) - return Command.Create(result.GetMetadata().ToPreparedDescriptor()); + { + var metadata = result.GetMetadata(); + return Command.Create(CommandDescriptor.CreatePrepared( + metadata.CommandName, + metadata.ParameterTypes.Preserve(), + metadata.RowDescription?.Preserve())); + } throw new InvalidOperationException("PostgreSQL preparation returned no command result."); } diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs index 70b75a0..59ef82a 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs @@ -25,9 +25,10 @@ public static ValueTask CreateAsync(IConfiguration configuratio return (database, driver) switch { - ("postgresql", "slon") => SlonFortuneDatabase.CreateAsync( + ("postgresql", "slon") => CreateSlonAsync( connectionString, - connectionCount), + connectionCount, + configuration["SLON_POOL_MODE"]), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(connectionString, connectionCount)), _ => throw new InvalidOperationException("The database selection is invalid."), @@ -77,31 +78,57 @@ private static int PositiveSetting(IConfiguration configuration, string name) ? parsed : throw new InvalidOperationException($"{name} must be a positive integer."); } + + static ValueTask CreateSlonAsync( + string connectionString, int connectionCount, string? configuredMode) + { + var mode = string.IsNullOrWhiteSpace(configuredMode) + ? "raw" + : configuredMode.Trim().ToLowerInvariant(); + Console.WriteLine($"Slon pool mode: {mode}."); + return mode switch + { + "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), + "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), + _ => throw new ArgumentOutOfRangeException( + "SLON_POOL_MODE", configuredMode, "Expected 'raw' or 'connection'."), + }; + } } -internal sealed class SlonFortuneDatabase : FortuneDatabase +internal sealed class RawSlonFortuneDatabase(RawSlonProtocolPool pool) : FortuneDatabase { - private readonly RawSlonProtocolPool _pool; - - private SlonFortuneDatabase(RawSlonProtocolPool pool) => _pool = pool; - public static async ValueTask CreateAsync( - string connectionString, - int connectionCount) - { - return new SlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( + string connectionString, int connectionCount) + => new RawSlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( connectionString, connectionCount).ConfigureAwait(false)); - } public override async ValueTask> LoadAsync( CancellationToken cancellationToken) { - var fortunes = await _pool.LoadAsync( + var fortunes = await pool.LoadAsync( + static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); + return Complete(fortunes); + } + + public override ValueTask DisposeAsync() => pool.DisposeAsync(); +} + +internal sealed class ConnectionSlonFortuneDatabase(FullSlonConnectionPool pool) : FortuneDatabase +{ + public static async ValueTask CreateAsync( + string connectionString, int connectionCount) + => new ConnectionSlonFortuneDatabase(await FullSlonConnectionPool.CreateAsync( + connectionString, connectionCount).ConfigureAwait(false)); + + public override async ValueTask> LoadAsync(CancellationToken cancellationToken) + { + var fortunes = await pool.LoadAsync( static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); return Complete(fortunes); } - public override ValueTask DisposeAsync() => _pool.DisposeAsync(); + public override ValueTask DisposeAsync() => pool.DisposeAsync(); } internal sealed class NpgsqlFortuneDatabase : FortuneDatabase diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md index 11f7c5e..5065511 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -14,6 +14,7 @@ Set the following configuration values as environment variables or equivalent .N | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | +| `SLON_POOL_MODE` | `raw` (default) or `connection` | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -21,12 +22,12 @@ unmerged branch. ## Driver strategies -Slon uses its experimental lower layer directly. A benchmark-local fixed pool opens -`DATABASE_CONNECTIONS` `PgClientProtocol` instances, prepares the statement once on every wire, -and places flows by atomic round-robin. This deliberately simple outer pool isolates Slon's -protocol/flow baseline. It does not exercise the richer production `SlonDataSource` placement -policy. -The raw Slon arm disables zero-byte reads to match Apex's ordinary BCL transport shape. +Slon uses its experimental lower layer directly in both modes, and creates a fresh +`ReaderDrivenCommandFlow` per request. `raw` opens `DATABASE_CONNECTIONS` protocols and places +flows by atomic round-robin. `connection` wraps the same protocols in `ConnectionPool` through +the lower-layer `IPoolConnection` seam, exercising production placement without adding ADO. +Every wire receives the same prepared statement before it becomes schedulable. +Both Slon modes disable zero-byte reads to match Apex's ordinary BCL transport shape. Npgsql uses a slim data source and a command bound to each leased connection. Both drivers materialize messages as strings, append and ordinally sort the same model, and render the same diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj index 019cd04..5926834 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj @@ -5,12 +5,13 @@ enable enable false - $(NoWarn);SLONPG001 + $(NoWarn);SLONPG001;SLONPOOL001 + diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs index 5231a88..2f6fe11 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs @@ -28,9 +28,10 @@ public static ValueTask CreateAsync( return (selectedDatabase, selectedDriver) switch { - ("postgresql", "slon") => SlonFortuneDatabase.CreateAsync( + ("postgresql", "slon") => CreateSlonAsync( requiredConnectionString, - connectionCount), + connectionCount, + Environment.GetEnvironmentVariable("SLON_POOL_MODE")), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(requiredConnectionString, connectionCount)), @@ -69,31 +70,57 @@ private static string RequiredSelection(string name, string? value) => string.IsNullOrWhiteSpace(value) ? throw new InvalidOperationException($"{name} is required.") : value.Trim().ToLowerInvariant(); + + static ValueTask CreateSlonAsync( + string connectionString, int connectionCount, string? configuredMode) + { + var mode = string.IsNullOrWhiteSpace(configuredMode) + ? "raw" + : configuredMode.Trim().ToLowerInvariant(); + Console.WriteLine($"Slon pool mode: {mode}."); + return mode switch + { + "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), + "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), + _ => throw new ArgumentOutOfRangeException( + "SLON_POOL_MODE", configuredMode, "Expected 'raw' or 'connection'."), + }; + } } -internal sealed class SlonFortuneDatabase : FortuneDatabase +internal sealed class RawSlonFortuneDatabase(RawSlonProtocolPool pool) : FortuneDatabase { - private readonly RawSlonProtocolPool _pool; - - private SlonFortuneDatabase(RawSlonProtocolPool pool) => _pool = pool; - public static async ValueTask CreateAsync( - string connectionString, - int connectionCount) - { - return new SlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( + string connectionString, int connectionCount) + => new RawSlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( connectionString, connectionCount).ConfigureAwait(false)); - } public override async ValueTask> LoadAsync( CancellationToken cancellationToken) { - var fortunes = await _pool.LoadAsync( + var fortunes = await pool.LoadAsync( + static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); + return Complete(fortunes); + } + + public override ValueTask DisposeAsync() => pool.DisposeAsync(); +} + +internal sealed class ConnectionSlonFortuneDatabase(FullSlonConnectionPool pool) : FortuneDatabase +{ + public static async ValueTask CreateAsync( + string connectionString, int connectionCount) + => new ConnectionSlonFortuneDatabase(await FullSlonConnectionPool.CreateAsync( + connectionString, connectionCount).ConfigureAwait(false)); + + public override async ValueTask> LoadAsync(CancellationToken cancellationToken) + { + var fortunes = await pool.LoadAsync( static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); return Complete(fortunes); } - public override ValueTask DisposeAsync() => _pool.DisposeAsync(); + public override ValueTask DisposeAsync() => pool.DisposeAsync(); } internal sealed class NpgsqlFortuneDatabase : FortuneDatabase diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index a477bd2..6c46abb 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -14,6 +14,7 @@ Set all of these environment variables before starting the app: | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | +| `SLON_POOL_MODE` | `raw` (default) or `connection` | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -21,12 +22,12 @@ unmerged branch. ## Driver strategies -Slon uses its experimental lower layer directly. A benchmark-local fixed pool opens -`DATABASE_CONNECTIONS` `PgClientProtocol` instances, prepares the statement once on every wire, -and places flows by atomic round-robin. This deliberately simple outer pool isolates Slon's -protocol/flow baseline. It does not exercise the richer production `SlonDataSource` placement -policy. -The raw Slon arm disables zero-byte reads to match Apex's ordinary BCL transport shape. +Slon uses its experimental lower layer directly in both modes, and creates a fresh +`ReaderDrivenCommandFlow` per request. `raw` opens `DATABASE_CONNECTIONS` protocols and places +flows by atomic round-robin. `connection` wraps the same protocols in `ConnectionPool` through +the lower-layer `IPoolConnection` seam, exercising production placement without adding ADO. +Every wire receives the same prepared statement before it becomes schedulable. +Both Slon modes disable zero-byte reads to match Apex's ordinary BCL transport shape. Npgsql uses a slim data source and a command bound to each leased connection. Both drivers materialize messages as strings, append and ordinally sort the same model, and render the same diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj index 8cc3a6e..0dac500 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj @@ -6,12 +6,13 @@ enable false $(DefineConstants);DATABASE - $(NoWarn);SLONPG001 + $(NoWarn);SLONPG001;SLONPOOL001 + From 16f3095b9388baacdc7bdf63afe09c25ad452316 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 29 Aug 2026 00:18:00 +0200 Subject: [PATCH 128/136] Benchmark streaming and collected Slon results --- .../Shared/FullSlonConnectionPool.cs | 55 ++++++++++++------ Slon.Benchmarks/Shared/RawSlonProtocolPool.cs | 57 +++++++++++++------ .../Slon.Fortunes.Minimal/FortuneDatabase.cs | 40 ++++++++----- .../Slon.Fortunes.Minimal/README.md | 2 + .../Slon.Fortunes.Platform/FortuneDatabase.cs | 40 ++++++++----- .../Slon.Fortunes.Platform/README.md | 2 + 6 files changed, 135 insertions(+), 61 deletions(-) diff --git a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs index d092c96..6ebba80 100644 --- a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs @@ -13,14 +13,18 @@ internal sealed class FullSlonConnectionPool : IAsyncDisposable { const string Query = "SELECT id, message FROM fortune"; readonly ConnectionPool _pool; - readonly Command _command; + readonly ReaderDrivenCommandOptions _options; + readonly SlonConsumptionMode _consumptionMode; - FullSlonConnectionPool(ConnectionPool pool, Command command) - => (_pool, _command) = (pool, command); + FullSlonConnectionPool(ConnectionPool pool, Command command, + SlonConsumptionMode consumptionMode) + => (_pool, _options, _consumptionMode) = + (pool, new ReaderDrivenCommandOptions(command), consumptionMode); internal static async ValueTask CreateAsync( string connectionString, - int connectionCount) + int connectionCount, + SlonConsumptionMode consumptionMode) { var builder = new NpgsqlConnectionStringBuilder(connectionString); var clientOptions = new PgClientOptions @@ -53,14 +57,14 @@ internal static async ValueTask CreateAsync( MaxConnections = connectionCount, ConnectionIdleLifetime = Timeout.InfiniteTimeSpan, }); - return new(pool, command); + return new(pool, command, consumptionMode); } public async ValueTask> LoadAsync( Func create, CancellationToken cancellationToken) { - var flow = new ReaderDrivenCommandFlow(_command); + var flow = new ReaderDrivenCommandFlow(_options); await _pool.GetAsync( static (candidate, item) => candidate.Connection.Protocol.TryQueue( item, @@ -72,10 +76,21 @@ await _pool.GetAsync( Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); - List values = []; - await foreach (var result in flow) - await foreach (var row in result) - values.Add(create(row.GetValue(0), row.GetValue(1))); + var values = new CollectList(create); + if (_consumptionMode is SlonConsumptionMode.Collect) + { + await flow.CollectAsync(values, static (state, row) => + { + var list = (CollectList)state!; + list.Add(list.Create(row.GetValue(0), row.GetValue(1))); + }, cancellationToken).ConfigureAwait(false); + } + else + { + await foreach (var result in flow) + await foreach (var row in result) + values.Add(create(row.GetValue(0), row.GetValue(1))); + } return values; } @@ -83,22 +98,21 @@ await _pool.GetAsync( static async ValueTask PrepareAsync(PgClientProtocol protocol) { - var command = Command.Create(Query, commandName: new EncodedCString("fortunes")) with - { - DescribeOnly = true, - DescribeForPreparation = true, - WithSync = true, - }; + var command = Command.Create(Query, commandName: new EncodedCString("fortunes")); var flow = protocol.Queue(new CommandFlow(async: true, command)); + Command? prepared = null; await foreach (var result in flow) { var metadata = result.GetMetadata(); - return Command.Create(CommandDescriptor.CreatePrepared( + prepared = Command.Create(CommandDescriptor.CreatePrepared( metadata.CommandName, metadata.ParameterTypes.Preserve(), metadata.RowDescription?.Preserve())); + await foreach (var _ in result) { } + _ = result.GetCommandComplete(); } - throw new InvalidOperationException("PostgreSQL preparation returned no command result."); + return prepared ?? + throw new InvalidOperationException("PostgreSQL preparation returned no command result."); } static string RequiredPostgreSqlValue(string name, string? value) @@ -159,4 +173,9 @@ public async ValueTask CreateAsync( } } } + + sealed class CollectList(Func create) : List + { + internal Func Create { get; } = create; + } } diff --git a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs index b84a17e..65892ea 100644 --- a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs +++ b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs @@ -12,13 +12,16 @@ internal sealed class RawSlonProtocolPool : IAsyncDisposable { const string Query = "SELECT id, message FROM fortune"; readonly Slot[] _slots; + readonly SlonConsumptionMode _consumptionMode; int _nextSlot = -1; - RawSlonProtocolPool(Slot[] slots) => _slots = slots; + RawSlonProtocolPool(Slot[] slots, SlonConsumptionMode consumptionMode) + => (_slots, _consumptionMode) = (slots, consumptionMode); internal static async ValueTask CreateAsync( string connectionString, - int connectionCount) + int connectionCount, + SlonConsumptionMode consumptionMode) { var builder = new NpgsqlConnectionStringBuilder(connectionString); var clientOptions = new PgClientOptions @@ -47,7 +50,8 @@ internal static async ValueTask CreateAsync( var protocol = await factory.CreateAsync().ConfigureAwait(false); try { - slots[created] = new(protocol, await PrepareAsync(protocol).ConfigureAwait(false)); + var command = await PrepareAsync(protocol).ConfigureAwait(false); + slots[created] = new(protocol, new ReaderDrivenCommandOptions(command)); } catch { @@ -55,7 +59,7 @@ internal static async ValueTask CreateAsync( throw; } } - return new(slots); + return new(slots, consumptionMode); } catch { @@ -70,13 +74,22 @@ public async ValueTask> LoadAsync( CancellationToken cancellationToken) { var slot = GetSlot(); - var flow = new ReaderDrivenCommandFlow(slot.Command); + var flow = new ReaderDrivenCommandFlow(slot.Options); if (!slot.Protocol.TryQueue(flow, cancellationToken: cancellationToken)) throw new InvalidOperationException("The selected PostgreSQL protocol is unavailable."); - List values = []; - await foreach (var result in flow) + var values = new CollectList(create); + if (_consumptionMode is SlonConsumptionMode.Collect) + { + await flow.CollectAsync(values, static (state, row) => + { + var list = (CollectList)state!; + list.Add(list.Create(row.GetValue(0), row.GetValue(1))); + }, cancellationToken).ConfigureAwait(false); + } + else { + await foreach (var result in flow) await foreach (var row in result) values.Add(create(row.GetValue(0), row.GetValue(1))); } @@ -106,22 +119,21 @@ public async ValueTask DisposeAsync() static async ValueTask PrepareAsync(PgClientProtocol protocol) { - var command = Command.Create(Query, commandName: new EncodedCString("fortunes")) with - { - DescribeOnly = true, - DescribeForPreparation = true, - WithSync = true, - }; + var command = Command.Create(Query, commandName: new EncodedCString("fortunes")); var flow = protocol.Queue(new CommandFlow(async: true, command)); + Command? prepared = null; await foreach (var result in flow) { var metadata = result.GetMetadata(); - return Command.Create(CommandDescriptor.CreatePrepared( + prepared = Command.Create(CommandDescriptor.CreatePrepared( metadata.CommandName, metadata.ParameterTypes.Preserve(), metadata.RowDescription?.Preserve())); + await foreach (var _ in result) { } + _ = result.GetCommandComplete(); } - throw new InvalidOperationException("PostgreSQL preparation returned no command result."); + return prepared ?? + throw new InvalidOperationException("PostgreSQL preparation returned no command result."); } static string RequiredPostgreSqlValue(string name, string? value) => @@ -129,9 +141,20 @@ static string RequiredPostgreSqlValue(string name, string? value) => ? throw new InvalidOperationException($"PostgreSQL {name} is required.") : value; - sealed class Slot(PgClientProtocol protocol, Command command) + sealed class Slot(PgClientProtocol protocol, ReaderDrivenCommandOptions options) { internal PgClientProtocol Protocol { get; } = protocol; - internal Command Command { get; } = command; + internal ReaderDrivenCommandOptions Options { get; } = options; } + + sealed class CollectList(Func create) : List + { + internal Func Create { get; } = create; + } +} + +internal enum SlonConsumptionMode +{ + Stream, + Collect, } diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs index 59ef82a..ab5577d 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs @@ -28,7 +28,8 @@ public static ValueTask CreateAsync(IConfiguration configuratio ("postgresql", "slon") => CreateSlonAsync( connectionString, connectionCount, - configuration["SLON_POOL_MODE"]), + configuration["SLON_POOL_MODE"], + configuration["SLON_CONSUMPTION_MODE"]), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(connectionString, connectionCount)), _ => throw new InvalidOperationException("The database selection is invalid."), @@ -80,28 +81,41 @@ private static int PositiveSetting(IConfiguration configuration, string name) } static ValueTask CreateSlonAsync( - string connectionString, int connectionCount, string? configuredMode) + string connectionString, int connectionCount, string? configuredPoolMode, + string? configuredConsumptionMode) { - var mode = string.IsNullOrWhiteSpace(configuredMode) + var poolMode = string.IsNullOrWhiteSpace(configuredPoolMode) ? "raw" - : configuredMode.Trim().ToLowerInvariant(); - Console.WriteLine($"Slon pool mode: {mode}."); - return mode switch + : configuredPoolMode.Trim().ToLowerInvariant(); + var consumptionMode = ParseConsumptionMode(configuredConsumptionMode); + Console.WriteLine($"Slon pool mode: {poolMode}; consumption mode: {consumptionMode.ToString().ToLowerInvariant()}."); + return poolMode switch { - "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), - "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), + "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), + "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), _ => throw new ArgumentOutOfRangeException( - "SLON_POOL_MODE", configuredMode, "Expected 'raw' or 'connection'."), + "SLON_POOL_MODE", configuredPoolMode, "Expected 'raw' or 'connection'."), }; } + + static SlonConsumptionMode ParseConsumptionMode(string? configuredMode) + => string.IsNullOrWhiteSpace(configuredMode) + ? SlonConsumptionMode.Stream + : configuredMode.Trim().ToLowerInvariant() switch + { + "stream" => SlonConsumptionMode.Stream, + "collect" => SlonConsumptionMode.Collect, + _ => throw new ArgumentOutOfRangeException( + "SLON_CONSUMPTION_MODE", configuredMode, "Expected 'stream' or 'collect'."), + }; } internal sealed class RawSlonFortuneDatabase(RawSlonProtocolPool pool) : FortuneDatabase { public static async ValueTask CreateAsync( - string connectionString, int connectionCount) + string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) => new RawSlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( - connectionString, connectionCount).ConfigureAwait(false)); + connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); public override async ValueTask> LoadAsync( CancellationToken cancellationToken) @@ -117,9 +131,9 @@ public override async ValueTask> LoadAsync( internal sealed class ConnectionSlonFortuneDatabase(FullSlonConnectionPool pool) : FortuneDatabase { public static async ValueTask CreateAsync( - string connectionString, int connectionCount) + string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) => new ConnectionSlonFortuneDatabase(await FullSlonConnectionPool.CreateAsync( - connectionString, connectionCount).ConfigureAwait(false)); + connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); public override async ValueTask> LoadAsync(CancellationToken cancellationToken) { diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md index 5065511..dd75ea8 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -15,6 +15,7 @@ Set the following configuration values as environment variables or equivalent .N | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | | `SLON_POOL_MODE` | `raw` (default) or `connection` | +| `SLON_CONSUMPTION_MODE` | `stream` (default) or `collect` | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -27,6 +28,7 @@ Slon uses its experimental lower layer directly in both modes, and creates a fre flows by atomic round-robin. `connection` wraps the same protocols in `ConnectionPool` through the lower-layer `IPoolConnection` seam, exercising production placement without adding ADO. Every wire receives the same prepared statement before it becomes schedulable. +`SLON_CONSUMPTION_MODE` independently selects nested streaming enumeration or one-await collection. Both Slon modes disable zero-byte reads to match Apex's ordinary BCL transport shape. Npgsql uses a slim data source and a command bound to each leased connection. Both drivers diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs index 2f6fe11..ee2daf1 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs @@ -31,7 +31,8 @@ public static ValueTask CreateAsync( ("postgresql", "slon") => CreateSlonAsync( requiredConnectionString, connectionCount, - Environment.GetEnvironmentVariable("SLON_POOL_MODE")), + Environment.GetEnvironmentVariable("SLON_POOL_MODE"), + Environment.GetEnvironmentVariable("SLON_CONSUMPTION_MODE")), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(requiredConnectionString, connectionCount)), @@ -72,28 +73,41 @@ private static string RequiredSelection(string name, string? value) => : value.Trim().ToLowerInvariant(); static ValueTask CreateSlonAsync( - string connectionString, int connectionCount, string? configuredMode) + string connectionString, int connectionCount, string? configuredPoolMode, + string? configuredConsumptionMode) { - var mode = string.IsNullOrWhiteSpace(configuredMode) + var poolMode = string.IsNullOrWhiteSpace(configuredPoolMode) ? "raw" - : configuredMode.Trim().ToLowerInvariant(); - Console.WriteLine($"Slon pool mode: {mode}."); - return mode switch + : configuredPoolMode.Trim().ToLowerInvariant(); + var consumptionMode = ParseConsumptionMode(configuredConsumptionMode); + Console.WriteLine($"Slon pool mode: {poolMode}; consumption mode: {consumptionMode.ToString().ToLowerInvariant()}."); + return poolMode switch { - "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), - "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount), + "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), + "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), _ => throw new ArgumentOutOfRangeException( - "SLON_POOL_MODE", configuredMode, "Expected 'raw' or 'connection'."), + "SLON_POOL_MODE", configuredPoolMode, "Expected 'raw' or 'connection'."), }; } + + static SlonConsumptionMode ParseConsumptionMode(string? configuredMode) + => string.IsNullOrWhiteSpace(configuredMode) + ? SlonConsumptionMode.Stream + : configuredMode.Trim().ToLowerInvariant() switch + { + "stream" => SlonConsumptionMode.Stream, + "collect" => SlonConsumptionMode.Collect, + _ => throw new ArgumentOutOfRangeException( + "SLON_CONSUMPTION_MODE", configuredMode, "Expected 'stream' or 'collect'."), + }; } internal sealed class RawSlonFortuneDatabase(RawSlonProtocolPool pool) : FortuneDatabase { public static async ValueTask CreateAsync( - string connectionString, int connectionCount) + string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) => new RawSlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( - connectionString, connectionCount).ConfigureAwait(false)); + connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); public override async ValueTask> LoadAsync( CancellationToken cancellationToken) @@ -109,9 +123,9 @@ public override async ValueTask> LoadAsync( internal sealed class ConnectionSlonFortuneDatabase(FullSlonConnectionPool pool) : FortuneDatabase { public static async ValueTask CreateAsync( - string connectionString, int connectionCount) + string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) => new ConnectionSlonFortuneDatabase(await FullSlonConnectionPool.CreateAsync( - connectionString, connectionCount).ConfigureAwait(false)); + connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); public override async ValueTask> LoadAsync(CancellationToken cancellationToken) { diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index 6c46abb..1557a9b 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -15,6 +15,7 @@ Set all of these environment variables before starting the app: | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | | `SLON_POOL_MODE` | `raw` (default) or `connection` | +| `SLON_CONSUMPTION_MODE` | `stream` (default) or `collect` | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -27,6 +28,7 @@ Slon uses its experimental lower layer directly in both modes, and creates a fre flows by atomic round-robin. `connection` wraps the same protocols in `ConnectionPool` through the lower-layer `IPoolConnection` seam, exercising production placement without adding ADO. Every wire receives the same prepared statement before it becomes schedulable. +`SLON_CONSUMPTION_MODE` independently selects nested streaming enumeration or one-await collection. Both Slon modes disable zero-byte reads to match Apex's ordinary BCL transport shape. Npgsql uses a slim data source and a command bound to each leased connection. Both drivers From 188cacef8a68fce9253102e482648533585b19b0 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Mon, 31 Aug 2026 15:03:07 +0200 Subject: [PATCH 129/136] Render fortunes from retained UTF-8 fields --- .../Shared/FullSlonConnectionPool.cs | 44 ++++++++++++++++++ Slon.Benchmarks/Shared/RawSlonProtocolPool.cs | 37 +++++++++++++++ .../BenchmarkApplication.cs | 8 ++-- .../Slon.Fortunes.Platform/Fortune.cs | 6 +-- .../Slon.Fortunes.Platform/FortuneDatabase.cs | 46 +++++++++++-------- .../Templates/Fortunes.cshtml | 2 +- 6 files changed, 118 insertions(+), 25 deletions(-) diff --git a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs index 6ebba80..29cea02 100644 --- a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs @@ -94,6 +94,50 @@ await flow.CollectAsync(values, static (state, row) => return values; } + public async ValueTask ConsumeRetainedAsync( + Func, T> create, + TState state, + Func, ValueTask> consume, + CancellationToken cancellationToken) + { + if (_consumptionMode is not SlonConsumptionMode.Stream) + throw new InvalidOperationException( + "Retained field memory requires streaming consumption."); + + var flow = new ReaderDrivenCommandFlow(_options); + await _pool.GetAsync( + static (candidate, item) => candidate.Connection.Protocol.TryQueue( + item, + candidate.IsIdleCandidate + ? FlowEnqueueOptions.None + : FlowEnqueueOptions.RequireExistingPipeline, + candidate.CancellationToken), + flow, + Timeout.InfiniteTimeSpan, + cancellationToken).ConfigureAwait(false); + + var values = new List(); + var results = flow.GetAsyncEnumerator(); + try + { + if (await results.MoveNextAsync().ConfigureAwait(false)) + { + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync().ConfigureAwait(false)) + { + var reader = rows.Current.GetReader(); + values.Add(create(reader.Read(), reader.ReadMemory())); + } + await rows.DisposeAsync().ConfigureAwait(false); + } + await consume(state, values).ConfigureAwait(false); + } + finally + { + await results.DisposeAsync().ConfigureAwait(false); + } + } + public ValueTask DisposeAsync() => _pool.DisposeAsync(); static async ValueTask PrepareAsync(PgClientProtocol protocol) diff --git a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs index 65892ea..f217858 100644 --- a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs +++ b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs @@ -96,6 +96,43 @@ await flow.CollectAsync(values, static (state, row) => return values; } + public async ValueTask ConsumeRetainedAsync( + Func, T> create, + TState state, + Func, ValueTask> consume, + CancellationToken cancellationToken) + { + if (_consumptionMode is not SlonConsumptionMode.Stream) + throw new InvalidOperationException( + "Retained field memory requires streaming consumption."); + + var slot = GetSlot(); + var flow = new ReaderDrivenCommandFlow(slot.Options); + if (!slot.Protocol.TryQueue(flow, cancellationToken: cancellationToken)) + throw new InvalidOperationException("The selected PostgreSQL protocol is unavailable."); + + var values = new List(); + var results = flow.GetAsyncEnumerator(); + try + { + if (await results.MoveNextAsync().ConfigureAwait(false)) + { + var rows = results.Current.GetAsyncEnumerator(); + while (await rows.MoveNextAsync().ConfigureAwait(false)) + { + var reader = rows.Current.GetReader(); + values.Add(create(reader.Read(), reader.ReadMemory())); + } + await rows.DisposeAsync().ConfigureAwait(false); + } + await consume(state, values).ConfigureAwait(false); + } + finally + { + await results.DisposeAsync().ConfigureAwait(false); + } + } + Slot GetSlot() => _slots[(int)((uint)Interlocked.Increment(ref _nextSlot) % (uint)_slots.Length)]; diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs index 4d25db7..2a2f07d 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs @@ -35,10 +35,12 @@ public void OnStartLine( }; private async Task RenderDatabaseAsync() + => await Database.RenderAsync(this, default); + + internal ValueTask RenderFortunesAsync(List fortunes) { - var template = Templates.Fortunes.Create( - await Database.LoadAsync(ConnectionClosed)); - await OutputFortunesAsync(Writer, template); + var template = Templates.Fortunes.Create(fortunes); + return OutputFortunesAsync(Writer, template); } private ValueTask OutputFortunesAsync( diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs index 2c26aef..2661ca9 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Fortune.cs @@ -2,7 +2,7 @@ namespace Slon.Fortunes.Platform; public readonly struct Fortune : IComparable { - public Fortune(int id, string message) + public Fortune(int id, ReadOnlyMemory message) { Id = id; Message = message; @@ -10,8 +10,8 @@ public Fortune(int id, string message) public int Id { get; } - public string Message { get; } + public ReadOnlyMemory Message { get; } public int CompareTo(Fortune other) => - StringComparer.Ordinal.Compare(Message, other.Message); + Message.Span.SequenceCompareTo(other.Message.Span); } diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs index ee2daf1..a577b57 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Text; using Npgsql; using Slon.Fortunes; @@ -7,11 +8,13 @@ namespace Slon.Fortunes.Platform; internal abstract class FortuneDatabase : IAsyncDisposable { internal const string Query = "SELECT id, message FROM fortune"; - private const string AdditionalFortune = "Additional fortune added at request time."; + private static readonly ReadOnlyMemory AdditionalFortune = + "Additional fortune added at request time."u8.ToArray(); public abstract ValueTask DisposeAsync(); - public abstract ValueTask> LoadAsync( + public abstract ValueTask RenderAsync( + BenchmarkApplication application, CancellationToken cancellationToken); public static ValueTask CreateAsync( @@ -109,13 +112,15 @@ public static async ValueTask CreateAsync( => new RawSlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); - public override async ValueTask> LoadAsync( + public override ValueTask RenderAsync( + BenchmarkApplication application, CancellationToken cancellationToken) - { - var fortunes = await pool.LoadAsync( - static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); - return Complete(fortunes); - } + => pool.ConsumeRetainedAsync( + static (id, message) => new Fortune(id, message), + application, + static (application, fortunes) => + application.RenderFortunesAsync(Complete(fortunes)), + cancellationToken); public override ValueTask DisposeAsync() => pool.DisposeAsync(); } @@ -127,12 +132,15 @@ public static async ValueTask CreateAsync( => new ConnectionSlonFortuneDatabase(await FullSlonConnectionPool.CreateAsync( connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); - public override async ValueTask> LoadAsync(CancellationToken cancellationToken) - { - var fortunes = await pool.LoadAsync( - static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); - return Complete(fortunes); - } + public override ValueTask RenderAsync( + BenchmarkApplication application, + CancellationToken cancellationToken) + => pool.ConsumeRetainedAsync( + static (id, message) => new Fortune(id, message), + application, + static (application, fortunes) => + application.RenderFortunesAsync(Complete(fortunes)), + cancellationToken); public override ValueTask DisposeAsync() => pool.DisposeAsync(); } @@ -150,19 +158,21 @@ public NpgsqlFortuneDatabase(string connectionString, int connectionCount) _dataSource = new NpgsqlSlimDataSourceBuilder(builder.ConnectionString).Build(); } - public override async ValueTask> LoadAsync( + public override async ValueTask RenderAsync( + BenchmarkApplication application, CancellationToken cancellationToken) { await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); await using var command = new NpgsqlCommand(Query, connection); await using var reader = await command.ExecuteReaderAsync(cancellationToken); - List fortunes = []; + var fortunes = new List(); while (await reader.ReadAsync(cancellationToken)) { - fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); + fortunes.Add(new Fortune( + reader.GetInt32(0), Encoding.UTF8.GetBytes(reader.GetString(1)))); } - return Complete(fortunes); + await application.RenderFortunesAsync(Complete(fortunes)); } public override ValueTask DisposeAsync() => _dataSource.DisposeAsync(); diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml index 39a4cb9..b36d18a 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml @@ -1,2 +1,2 @@ @inherits RazorSlice> -Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message
+Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message.Span
From 21e84cbaf565a58a250234c8b4b0234156737303 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Mon, 31 Aug 2026 16:05:50 +0200 Subject: [PATCH 130/136] Tighten Fortunes consumption paths --- .../Shared/FullSlonConnectionPool.cs | 6 ++- Slon.Benchmarks/Shared/RawSlonProtocolPool.cs | 6 ++- .../BenchmarkApplication.cs | 11 ++-- .../Slon.Fortunes.Platform/FortuneDatabase.cs | 50 ++++++++++++++----- .../Slon.Fortunes.Platform/README.md | 15 +++--- 5 files changed, 58 insertions(+), 30 deletions(-) diff --git a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs index 29cea02..7418e52 100644 --- a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs @@ -21,6 +21,8 @@ internal sealed class FullSlonConnectionPool : IAsyncDisposable => (_pool, _options, _consumptionMode) = (pool, new ReaderDrivenCommandOptions(command), consumptionMode); + internal SlonConsumptionMode ConsumptionMode => _consumptionMode; + internal static async ValueTask CreateAsync( string connectionString, int connectionCount, @@ -87,7 +89,7 @@ await flow.CollectAsync(values, static (state, row) => } else { - await foreach (var result in flow) + await foreach (var result in flow.GetAsyncEnumerator(cancellationToken)) await foreach (var row in result) values.Add(create(row.GetValue(0), row.GetValue(1))); } @@ -117,7 +119,7 @@ await _pool.GetAsync( cancellationToken).ConfigureAwait(false); var values = new List(); - var results = flow.GetAsyncEnumerator(); + var results = flow.GetAsyncEnumerator(cancellationToken); try { if (await results.MoveNextAsync().ConfigureAwait(false)) diff --git a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs index f217858..9b99dcd 100644 --- a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs +++ b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs @@ -18,6 +18,8 @@ internal sealed class RawSlonProtocolPool : IAsyncDisposable RawSlonProtocolPool(Slot[] slots, SlonConsumptionMode consumptionMode) => (_slots, _consumptionMode) = (slots, consumptionMode); + internal SlonConsumptionMode ConsumptionMode => _consumptionMode; + internal static async ValueTask CreateAsync( string connectionString, int connectionCount, @@ -89,7 +91,7 @@ await flow.CollectAsync(values, static (state, row) => } else { - await foreach (var result in flow) + await foreach (var result in flow.GetAsyncEnumerator(cancellationToken)) await foreach (var row in result) values.Add(create(row.GetValue(0), row.GetValue(1))); } @@ -112,7 +114,7 @@ public async ValueTask ConsumeRetainedAsync( throw new InvalidOperationException("The selected PostgreSQL protocol is unavailable."); var values = new List(); - var results = flow.GetAsyncEnumerator(); + var results = flow.GetAsyncEnumerator(cancellationToken); try { if (await results.MoveNextAsync().ConfigureAwait(false)) diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs index 2a2f07d..794820e 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs @@ -28,15 +28,12 @@ public void OnStartLine( : RequestType.NotRecognized; } - private Task ProcessRequestAsync() => _requestType switch + private ValueTask ProcessRequestAsync() => _requestType switch { - RequestType.Fortunes => RenderDatabaseAsync(), + RequestType.Fortunes => Database.RenderAsync(this, default), _ => OutputEmptyAsync(Writer), }; - private async Task RenderDatabaseAsync() - => await Database.RenderAsync(this, default); - internal ValueTask RenderFortunesAsync(List fortunes) { var template = Templates.Fortunes.Create(fortunes); @@ -59,12 +56,12 @@ private ValueTask OutputFortunesAsync( return AwaitTemplateRenderTask(renderTask, chunkedWriter, template); } - private static Task OutputEmptyAsync(PipeWriter pipeWriter) + private static ValueTask OutputEmptyAsync(PipeWriter pipeWriter) { var writer = StartResponse(pipeWriter); writer.Complete(); ReturnChunkedWriter(writer); - return Task.CompletedTask; + return ValueTask.CompletedTask; } private static ChunkedPipeWriter StartResponse(PipeWriter pipeWriter) diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs index a577b57..24374b4 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs @@ -115,12 +115,25 @@ public static async ValueTask CreateAsync( public override ValueTask RenderAsync( BenchmarkApplication application, CancellationToken cancellationToken) - => pool.ConsumeRetainedAsync( - static (id, message) => new Fortune(id, message), - application, - static (application, fortunes) => - application.RenderFortunesAsync(Complete(fortunes)), - cancellationToken); + => pool.ConsumptionMode is SlonConsumptionMode.Stream + ? pool.ConsumeRetainedAsync( + static (id, message) => new Fortune(id, message), + application, + static (application, fortunes) => + application.RenderFortunesAsync(Complete(fortunes)), + cancellationToken) + : RenderCollectedAsync(pool, application, cancellationToken); + + static async ValueTask RenderCollectedAsync( + RawSlonProtocolPool pool, + BenchmarkApplication application, + CancellationToken cancellationToken) + { + var fortunes = await pool.LoadAsync( + static (id, message) => new Fortune(id, Encoding.UTF8.GetBytes(message)), + cancellationToken).ConfigureAwait(false); + await application.RenderFortunesAsync(Complete(fortunes)).ConfigureAwait(false); + } public override ValueTask DisposeAsync() => pool.DisposeAsync(); } @@ -135,12 +148,25 @@ public static async ValueTask CreateAsync( public override ValueTask RenderAsync( BenchmarkApplication application, CancellationToken cancellationToken) - => pool.ConsumeRetainedAsync( - static (id, message) => new Fortune(id, message), - application, - static (application, fortunes) => - application.RenderFortunesAsync(Complete(fortunes)), - cancellationToken); + => pool.ConsumptionMode is SlonConsumptionMode.Stream + ? pool.ConsumeRetainedAsync( + static (id, message) => new Fortune(id, message), + application, + static (application, fortunes) => + application.RenderFortunesAsync(Complete(fortunes)), + cancellationToken) + : RenderCollectedAsync(pool, application, cancellationToken); + + static async ValueTask RenderCollectedAsync( + FullSlonConnectionPool pool, + BenchmarkApplication application, + CancellationToken cancellationToken) + { + var fortunes = await pool.LoadAsync( + static (id, message) => new Fortune(id, Encoding.UTF8.GetBytes(message)), + cancellationToken).ConfigureAwait(false); + await application.RenderFortunesAsync(Complete(fortunes)).ConfigureAwait(false); + } public override ValueTask DisposeAsync() => pool.DisposeAsync(); } diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index 1557a9b..2cd0400 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -27,13 +27,14 @@ Slon uses its experimental lower layer directly in both modes, and creates a fre `ReaderDrivenCommandFlow` per request. `raw` opens `DATABASE_CONNECTIONS` protocols and places flows by atomic round-robin. `connection` wraps the same protocols in `ConnectionPool` through the lower-layer `IPoolConnection` seam, exercising production placement without adding ADO. -Every wire receives the same prepared statement before it becomes schedulable. -`SLON_CONSUMPTION_MODE` independently selects nested streaming enumeration or one-await collection. -Both Slon modes disable zero-byte reads to match Apex's ordinary BCL transport shape. - -Npgsql uses a slim data source and a command bound to each leased connection. Both drivers -materialize messages as strings, append and ordinally sort the same model, and render the same -RazorSlices string template for a fair comparison. +Every wire receives the same prepared statement before it becomes schedulable. In `stream` mode, +the response retains UTF-8 field memory through rendering, avoiding per-row strings and byte arrays. +`collect` exercises the one-await collector and materializes strings before rendering. Both Slon +pool modes disable zero-byte reads to match Apex's ordinary BCL transport shape. + +Npgsql uses a slim data source and a command bound to each leased connection. Every strategy +appends and ordinally sorts the same logical model and renders through the same RazorSlices UTF-8 +template. The Crank configuration uses two fewer Slon connections than database cores and 256 Npgsql connections. From 104204881756ead3fce434d3103d3b6a340d76d3 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Mon, 31 Aug 2026 16:36:52 +0200 Subject: [PATCH 131/136] Drive pooled protocol heartbeats centrally --- .../Shared/FullSlonConnectionPool.cs | 59 ++++++++++++++++--- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs index 7418e52..0389a1c 100644 --- a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Net; using Npgsql; using Slon.Pg; @@ -38,17 +39,19 @@ internal static async ValueTask CreateAsync( Password = builder.Password, Ssl = new PostgreSqlSslOptions { Mode = PostgreSqlSslMode.Disable }, }; + var transportFactory = SocketStreamConnection.CreateFactory( + clientOptions.EndPoint, + new TransportConnectionOptions { UseZeroByteReads = false }); + var bootstrapFactory = new PgClientProtocolFactory(clientOptions, transportFactory); var protocolFactory = new PgClientProtocolFactory( clientOptions, - SocketStreamConnection.CreateFactory(clientOptions.EndPoint, new TransportConnectionOptions - { - UseZeroByteReads = false, - })); + transportFactory, + static options => options.HeartbeatMode = PgClientProtocolHeartbeatMode.External); // Every pooled wire installs the same named statement. Obtain its immutable descriptor once; // later flows can be created before placement and use it on whichever wire the pool selects. Command command; - await using (var bootstrap = await protocolFactory.CreateAsync().ConfigureAwait(false)) + await using (var bootstrap = await bootstrapFactory.CreateAsync().ConfigureAwait(false)) command = await PrepareAsync(bootstrap).ConfigureAwait(false); var pool = new ConnectionPool( @@ -169,17 +172,51 @@ static string RequiredPostgreSqlValue(string name, string? value) sealed class ProtocolConnection(PgClientProtocol protocol) : IPoolConnection { + IDisposable? _heartbeatRegistration; + internal PgClientProtocol Protocol { get; } = protocol; public bool IsIdle => Protocol.Outstanding == 0; public bool IsSchedulable => Protocol.IsSchedulable; public Task Completion => Protocol.Completion; - public Task CompleteAsync(Exception? exception = null) => Protocol.CompleteAsync(exception); + public Task CompleteAsync(Exception? exception = null) + { + var completion = Protocol.CompleteAsync(exception); + if (completion.IsCompleted) + { + StopHeartbeat(); + return completion; + } + return CompleteAndStopHeartbeat(completion, this); + + static async Task CompleteAndStopHeartbeat( + Task completion, ProtocolConnection connection) + { + try + { + await completion.ConfigureAwait(false); + } + finally + { + connection.StopHeartbeat(); + } + } + } public int CompareTo(ProtocolConnection? other) => other is null ? 1 : Protocol.Outstanding.CompareTo(other.Protocol.Outstanding); public void Start(ConnectionPool.Registration registration) => Protocol.SetAdmissionAvailableCallback( () => registration.SignalAvailability(Protocol.Outstanding == 0)); + + internal void StartHeartbeat(ConnectionPoolContext poolContext) + { + Debug.Assert(_heartbeatRegistration is null); + _heartbeatRegistration = poolContext.OnHeartbeat( + static (connection, elapsed) => connection.Protocol.HeartbeatAsync(elapsed), this); + } + + internal void StopHeartbeat() + => Interlocked.Exchange(ref _heartbeatRegistration, null)?.Dispose(); } sealed class ProtocolConnectionFactory(PgClientProtocolFactory factory) @@ -190,13 +227,16 @@ public ProtocolConnection Create( TimeSpan timeout = default) { var protocol = factory.Create(timeout); + var connection = new ProtocolConnection(protocol); + connection.StartHeartbeat(poolContext); try { _ = PrepareAsync(protocol).AsTask().GetAwaiter().GetResult(); - return new(protocol); + return connection; } catch { + connection.StopHeartbeat(); protocol.Dispose(); throw; } @@ -207,13 +247,16 @@ public async ValueTask CreateAsync( CancellationToken cancellationToken = default) { var protocol = await factory.CreateAsync(cancellationToken).ConfigureAwait(false); + var connection = new ProtocolConnection(protocol); + connection.StartHeartbeat(poolContext); try { _ = await PrepareAsync(protocol).ConfigureAwait(false); - return new(protocol); + return connection; } catch { + connection.StopHeartbeat(); await protocol.DisposeAsync().ConfigureAwait(false); throw; } From 374e2bd254a7a8ba7bd437a46752d93368f601a7 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Mon, 31 Aug 2026 17:24:50 +0200 Subject: [PATCH 132/136] Use one production Fortunes path --- Slon.Benchmarks/Shared/RawSlonProtocolPool.cs | 199 ------------------ ...onnectionPool.cs => SlonConnectionPool.cs} | 46 +--- .../Slon.Fortunes.Minimal/FortuneDatabase.cs | 61 +----- .../Slon.Fortunes.Minimal/README.md | 13 +- .../Slon.Fortunes.Minimal.csproj | 3 +- .../Slon.Fortunes.Platform/FortuneDatabase.cs | 101 ++------- .../Slon.Fortunes.Platform/README.md | 15 +- .../Slon.Fortunes.Platform.csproj | 3 +- 8 files changed, 39 insertions(+), 402 deletions(-) delete mode 100644 Slon.Benchmarks/Shared/RawSlonProtocolPool.cs rename Slon.Benchmarks/Shared/{FullSlonConnectionPool.cs => SlonConnectionPool.cs} (84%) diff --git a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs b/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs deleted file mode 100644 index 9b99dcd..0000000 --- a/Slon.Benchmarks/Shared/RawSlonProtocolPool.cs +++ /dev/null @@ -1,199 +0,0 @@ -using System.Net; -using Npgsql; -using Slon.Pg; -using Slon.Pg.Protocol; -using Slon.Pg.Protocol.Flows; -using Slon.Text; -using Slon.Transport; - -namespace Slon.Fortunes; - -internal sealed class RawSlonProtocolPool : IAsyncDisposable -{ - const string Query = "SELECT id, message FROM fortune"; - readonly Slot[] _slots; - readonly SlonConsumptionMode _consumptionMode; - int _nextSlot = -1; - - RawSlonProtocolPool(Slot[] slots, SlonConsumptionMode consumptionMode) - => (_slots, _consumptionMode) = (slots, consumptionMode); - - internal SlonConsumptionMode ConsumptionMode => _consumptionMode; - - internal static async ValueTask CreateAsync( - string connectionString, - int connectionCount, - SlonConsumptionMode consumptionMode) - { - var builder = new NpgsqlConnectionStringBuilder(connectionString); - var clientOptions = new PgClientOptions - { - EndPoint = new DnsEndPoint( - RequiredPostgreSqlValue("Host", builder.Host), - builder.Port), - Database = RequiredPostgreSqlValue("Database", builder.Database), - Username = RequiredPostgreSqlValue("Username", builder.Username), - Password = builder.Password, - Ssl = new PostgreSqlSslOptions { Mode = PostgreSqlSslMode.Disable }, - }; - var factory = new PgClientProtocolFactory( - clientOptions, - SocketStreamConnection.CreateFactory(clientOptions.EndPoint, new TransportConnectionOptions - { - // Match Apex's ordinary BCL read shape for this lower-layer ceiling comparison. - UseZeroByteReads = false, - })); - var slots = new Slot[connectionCount]; - var created = 0; - try - { - for (; created < slots.Length; created++) - { - var protocol = await factory.CreateAsync().ConfigureAwait(false); - try - { - var command = await PrepareAsync(protocol).ConfigureAwait(false); - slots[created] = new(protocol, new ReaderDrivenCommandOptions(command)); - } - catch - { - await protocol.DisposeAsync().ConfigureAwait(false); - throw; - } - } - return new(slots, consumptionMode); - } - catch - { - for (var i = 0; i < created; i++) - await slots[i].Protocol.DisposeAsync().ConfigureAwait(false); - throw; - } - } - - public async ValueTask> LoadAsync( - Func create, - CancellationToken cancellationToken) - { - var slot = GetSlot(); - var flow = new ReaderDrivenCommandFlow(slot.Options); - if (!slot.Protocol.TryQueue(flow, cancellationToken: cancellationToken)) - throw new InvalidOperationException("The selected PostgreSQL protocol is unavailable."); - - var values = new CollectList(create); - if (_consumptionMode is SlonConsumptionMode.Collect) - { - await flow.CollectAsync(values, static (state, row) => - { - var list = (CollectList)state!; - list.Add(list.Create(row.GetValue(0), row.GetValue(1))); - }, cancellationToken).ConfigureAwait(false); - } - else - { - await foreach (var result in flow.GetAsyncEnumerator(cancellationToken)) - await foreach (var row in result) - values.Add(create(row.GetValue(0), row.GetValue(1))); - } - return values; - } - - public async ValueTask ConsumeRetainedAsync( - Func, T> create, - TState state, - Func, ValueTask> consume, - CancellationToken cancellationToken) - { - if (_consumptionMode is not SlonConsumptionMode.Stream) - throw new InvalidOperationException( - "Retained field memory requires streaming consumption."); - - var slot = GetSlot(); - var flow = new ReaderDrivenCommandFlow(slot.Options); - if (!slot.Protocol.TryQueue(flow, cancellationToken: cancellationToken)) - throw new InvalidOperationException("The selected PostgreSQL protocol is unavailable."); - - var values = new List(); - var results = flow.GetAsyncEnumerator(cancellationToken); - try - { - if (await results.MoveNextAsync().ConfigureAwait(false)) - { - var rows = results.Current.GetAsyncEnumerator(); - while (await rows.MoveNextAsync().ConfigureAwait(false)) - { - var reader = rows.Current.GetReader(); - values.Add(create(reader.Read(), reader.ReadMemory())); - } - await rows.DisposeAsync().ConfigureAwait(false); - } - await consume(state, values).ConfigureAwait(false); - } - finally - { - await results.DisposeAsync().ConfigureAwait(false); - } - } - - Slot GetSlot() - => _slots[(int)((uint)Interlocked.Increment(ref _nextSlot) % (uint)_slots.Length)]; - - public async ValueTask DisposeAsync() - { - List? errors = null; - foreach (var slot in _slots) - { - try - { - await slot.Protocol.DisposeAsync().ConfigureAwait(false); - } - catch (Exception exception) - { - (errors ??= []).Add(exception); - } - } - if (errors is not null) - throw errors.Count is 1 ? errors[0] : new AggregateException(errors); - } - - static async ValueTask PrepareAsync(PgClientProtocol protocol) - { - var command = Command.Create(Query, commandName: new EncodedCString("fortunes")); - var flow = protocol.Queue(new CommandFlow(async: true, command)); - Command? prepared = null; - await foreach (var result in flow) - { - var metadata = result.GetMetadata(); - prepared = Command.Create(CommandDescriptor.CreatePrepared( - metadata.CommandName, - metadata.ParameterTypes.Preserve(), - metadata.RowDescription?.Preserve())); - await foreach (var _ in result) { } - _ = result.GetCommandComplete(); - } - return prepared ?? - throw new InvalidOperationException("PostgreSQL preparation returned no command result."); - } - - static string RequiredPostgreSqlValue(string name, string? value) => - string.IsNullOrWhiteSpace(value) - ? throw new InvalidOperationException($"PostgreSQL {name} is required.") - : value; - - sealed class Slot(PgClientProtocol protocol, ReaderDrivenCommandOptions options) - { - internal PgClientProtocol Protocol { get; } = protocol; - internal ReaderDrivenCommandOptions Options { get; } = options; - } - - sealed class CollectList(Func create) : List - { - internal Func Create { get; } = create; - } -} - -internal enum SlonConsumptionMode -{ - Stream, - Collect, -} diff --git a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs b/Slon.Benchmarks/Shared/SlonConnectionPool.cs similarity index 84% rename from Slon.Benchmarks/Shared/FullSlonConnectionPool.cs rename to Slon.Benchmarks/Shared/SlonConnectionPool.cs index 0389a1c..e34c1d0 100644 --- a/Slon.Benchmarks/Shared/FullSlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/SlonConnectionPool.cs @@ -10,24 +10,18 @@ namespace Slon.Fortunes; -internal sealed class FullSlonConnectionPool : IAsyncDisposable +internal sealed class SlonConnectionPool : IAsyncDisposable { const string Query = "SELECT id, message FROM fortune"; readonly ConnectionPool _pool; readonly ReaderDrivenCommandOptions _options; - readonly SlonConsumptionMode _consumptionMode; - FullSlonConnectionPool(ConnectionPool pool, Command command, - SlonConsumptionMode consumptionMode) - => (_pool, _options, _consumptionMode) = - (pool, new ReaderDrivenCommandOptions(command), consumptionMode); + SlonConnectionPool(ConnectionPool pool, Command command) + => (_pool, _options) = (pool, new ReaderDrivenCommandOptions(command)); - internal SlonConsumptionMode ConsumptionMode => _consumptionMode; - - internal static async ValueTask CreateAsync( + internal static async ValueTask CreateAsync( string connectionString, - int connectionCount, - SlonConsumptionMode consumptionMode) + int connectionCount) { var builder = new NpgsqlConnectionStringBuilder(connectionString); var clientOptions = new PgClientOptions @@ -62,7 +56,7 @@ internal static async ValueTask CreateAsync( MaxConnections = connectionCount, ConnectionIdleLifetime = Timeout.InfiniteTimeSpan, }); - return new(pool, command, consumptionMode); + return new(pool, command); } public async ValueTask> LoadAsync( @@ -81,21 +75,10 @@ await _pool.GetAsync( Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); - var values = new CollectList(create); - if (_consumptionMode is SlonConsumptionMode.Collect) - { - await flow.CollectAsync(values, static (state, row) => - { - var list = (CollectList)state!; - list.Add(list.Create(row.GetValue(0), row.GetValue(1))); - }, cancellationToken).ConfigureAwait(false); - } - else - { - await foreach (var result in flow.GetAsyncEnumerator(cancellationToken)) - await foreach (var row in result) - values.Add(create(row.GetValue(0), row.GetValue(1))); - } + var values = new List(); + await foreach (var result in flow.GetAsyncEnumerator(cancellationToken)) + await foreach (var row in result) + values.Add(create(row.GetValue(0), row.GetValue(1))); return values; } @@ -105,10 +88,6 @@ public async ValueTask ConsumeRetainedAsync( Func, ValueTask> consume, CancellationToken cancellationToken) { - if (_consumptionMode is not SlonConsumptionMode.Stream) - throw new InvalidOperationException( - "Retained field memory requires streaming consumption."); - var flow = new ReaderDrivenCommandFlow(_options); await _pool.GetAsync( static (candidate, item) => candidate.Connection.Protocol.TryQueue( @@ -262,9 +241,4 @@ public async ValueTask CreateAsync( } } } - - sealed class CollectList(Func create) : List - { - internal Func Create { get; } = create; - } } diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs index ab5577d..2d2123b 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs @@ -25,11 +25,8 @@ public static ValueTask CreateAsync(IConfiguration configuratio return (database, driver) switch { - ("postgresql", "slon") => CreateSlonAsync( - connectionString, - connectionCount, - configuration["SLON_POOL_MODE"], - configuration["SLON_CONSUMPTION_MODE"]), + ("postgresql", "slon") => + SlonFortuneDatabase.CreateAsync(connectionString, connectionCount), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(connectionString, connectionCount)), _ => throw new InvalidOperationException("The database selection is invalid."), @@ -80,60 +77,14 @@ private static int PositiveSetting(IConfiguration configuration, string name) : throw new InvalidOperationException($"{name} must be a positive integer."); } - static ValueTask CreateSlonAsync( - string connectionString, int connectionCount, string? configuredPoolMode, - string? configuredConsumptionMode) - { - var poolMode = string.IsNullOrWhiteSpace(configuredPoolMode) - ? "raw" - : configuredPoolMode.Trim().ToLowerInvariant(); - var consumptionMode = ParseConsumptionMode(configuredConsumptionMode); - Console.WriteLine($"Slon pool mode: {poolMode}; consumption mode: {consumptionMode.ToString().ToLowerInvariant()}."); - return poolMode switch - { - "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), - "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), - _ => throw new ArgumentOutOfRangeException( - "SLON_POOL_MODE", configuredPoolMode, "Expected 'raw' or 'connection'."), - }; - } - - static SlonConsumptionMode ParseConsumptionMode(string? configuredMode) - => string.IsNullOrWhiteSpace(configuredMode) - ? SlonConsumptionMode.Stream - : configuredMode.Trim().ToLowerInvariant() switch - { - "stream" => SlonConsumptionMode.Stream, - "collect" => SlonConsumptionMode.Collect, - _ => throw new ArgumentOutOfRangeException( - "SLON_CONSUMPTION_MODE", configuredMode, "Expected 'stream' or 'collect'."), - }; -} - -internal sealed class RawSlonFortuneDatabase(RawSlonProtocolPool pool) : FortuneDatabase -{ - public static async ValueTask CreateAsync( - string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) - => new RawSlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( - connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); - - public override async ValueTask> LoadAsync( - CancellationToken cancellationToken) - { - var fortunes = await pool.LoadAsync( - static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); - return Complete(fortunes); - } - - public override ValueTask DisposeAsync() => pool.DisposeAsync(); } -internal sealed class ConnectionSlonFortuneDatabase(FullSlonConnectionPool pool) : FortuneDatabase +internal sealed class SlonFortuneDatabase(SlonConnectionPool pool) : FortuneDatabase { public static async ValueTask CreateAsync( - string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) - => new ConnectionSlonFortuneDatabase(await FullSlonConnectionPool.CreateAsync( - connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); + string connectionString, int connectionCount) + => new SlonFortuneDatabase(await SlonConnectionPool.CreateAsync( + connectionString, connectionCount).ConfigureAwait(false)); public override async ValueTask> LoadAsync(CancellationToken cancellationToken) { diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md index dd75ea8..dd966e6 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -14,8 +14,6 @@ Set the following configuration values as environment variables or equivalent .N | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | -| `SLON_POOL_MODE` | `raw` (default) or `connection` | -| `SLON_CONSUMPTION_MODE` | `stream` (default) or `collect` | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -23,13 +21,10 @@ unmerged branch. ## Driver strategies -Slon uses its experimental lower layer directly in both modes, and creates a fresh -`ReaderDrivenCommandFlow` per request. `raw` opens `DATABASE_CONNECTIONS` protocols and places -flows by atomic round-robin. `connection` wraps the same protocols in `ConnectionPool` through -the lower-layer `IPoolConnection` seam, exercising production placement without adding ADO. -Every wire receives the same prepared statement before it becomes schedulable. -`SLON_CONSUMPTION_MODE` independently selects nested streaming enumeration or one-await collection. -Both Slon modes disable zero-byte reads to match Apex's ordinary BCL transport shape. +Slon uses its experimental lower layer through `ConnectionPool` and creates a fresh +`ReaderDrivenCommandFlow` per request. Every wire receives the same prepared statement before it +becomes schedulable. Results are consumed through nested streaming enumeration, and zero-byte reads +are disabled to match Apex's ordinary BCL transport shape. Npgsql uses a slim data source and a command bound to each leased connection. Both drivers materialize messages as strings, append and ordinally sort the same model, and render the same diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj index 5926834..274c2c6 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj @@ -10,8 +10,7 @@ - - + diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs index 24374b4..1a8d43a 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/FortuneDatabase.cs @@ -31,11 +31,8 @@ public static ValueTask CreateAsync( return (selectedDatabase, selectedDriver) switch { - ("postgresql", "slon") => CreateSlonAsync( - requiredConnectionString, - connectionCount, - Environment.GetEnvironmentVariable("SLON_POOL_MODE"), - Environment.GetEnvironmentVariable("SLON_CONSUMPTION_MODE")), + ("postgresql", "slon") => + SlonFortuneDatabase.CreateAsync(requiredConnectionString, connectionCount), ("postgresql", "npgsql") => ValueTask.FromResult( new NpgsqlFortuneDatabase(requiredConnectionString, connectionCount)), @@ -75,98 +72,24 @@ private static string RequiredSelection(string name, string? value) => ? throw new InvalidOperationException($"{name} is required.") : value.Trim().ToLowerInvariant(); - static ValueTask CreateSlonAsync( - string connectionString, int connectionCount, string? configuredPoolMode, - string? configuredConsumptionMode) - { - var poolMode = string.IsNullOrWhiteSpace(configuredPoolMode) - ? "raw" - : configuredPoolMode.Trim().ToLowerInvariant(); - var consumptionMode = ParseConsumptionMode(configuredConsumptionMode); - Console.WriteLine($"Slon pool mode: {poolMode}; consumption mode: {consumptionMode.ToString().ToLowerInvariant()}."); - return poolMode switch - { - "raw" => RawSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), - "connection" => ConnectionSlonFortuneDatabase.CreateAsync(connectionString, connectionCount, consumptionMode), - _ => throw new ArgumentOutOfRangeException( - "SLON_POOL_MODE", configuredPoolMode, "Expected 'raw' or 'connection'."), - }; - } - - static SlonConsumptionMode ParseConsumptionMode(string? configuredMode) - => string.IsNullOrWhiteSpace(configuredMode) - ? SlonConsumptionMode.Stream - : configuredMode.Trim().ToLowerInvariant() switch - { - "stream" => SlonConsumptionMode.Stream, - "collect" => SlonConsumptionMode.Collect, - _ => throw new ArgumentOutOfRangeException( - "SLON_CONSUMPTION_MODE", configuredMode, "Expected 'stream' or 'collect'."), - }; -} - -internal sealed class RawSlonFortuneDatabase(RawSlonProtocolPool pool) : FortuneDatabase -{ - public static async ValueTask CreateAsync( - string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) - => new RawSlonFortuneDatabase(await RawSlonProtocolPool.CreateAsync( - connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); - - public override ValueTask RenderAsync( - BenchmarkApplication application, - CancellationToken cancellationToken) - => pool.ConsumptionMode is SlonConsumptionMode.Stream - ? pool.ConsumeRetainedAsync( - static (id, message) => new Fortune(id, message), - application, - static (application, fortunes) => - application.RenderFortunesAsync(Complete(fortunes)), - cancellationToken) - : RenderCollectedAsync(pool, application, cancellationToken); - - static async ValueTask RenderCollectedAsync( - RawSlonProtocolPool pool, - BenchmarkApplication application, - CancellationToken cancellationToken) - { - var fortunes = await pool.LoadAsync( - static (id, message) => new Fortune(id, Encoding.UTF8.GetBytes(message)), - cancellationToken).ConfigureAwait(false); - await application.RenderFortunesAsync(Complete(fortunes)).ConfigureAwait(false); - } - - public override ValueTask DisposeAsync() => pool.DisposeAsync(); } -internal sealed class ConnectionSlonFortuneDatabase(FullSlonConnectionPool pool) : FortuneDatabase +internal sealed class SlonFortuneDatabase(SlonConnectionPool pool) : FortuneDatabase { public static async ValueTask CreateAsync( - string connectionString, int connectionCount, SlonConsumptionMode consumptionMode) - => new ConnectionSlonFortuneDatabase(await FullSlonConnectionPool.CreateAsync( - connectionString, connectionCount, consumptionMode).ConfigureAwait(false)); + string connectionString, int connectionCount) + => new SlonFortuneDatabase(await SlonConnectionPool.CreateAsync( + connectionString, connectionCount).ConfigureAwait(false)); public override ValueTask RenderAsync( BenchmarkApplication application, CancellationToken cancellationToken) - => pool.ConsumptionMode is SlonConsumptionMode.Stream - ? pool.ConsumeRetainedAsync( - static (id, message) => new Fortune(id, message), - application, - static (application, fortunes) => - application.RenderFortunesAsync(Complete(fortunes)), - cancellationToken) - : RenderCollectedAsync(pool, application, cancellationToken); - - static async ValueTask RenderCollectedAsync( - FullSlonConnectionPool pool, - BenchmarkApplication application, - CancellationToken cancellationToken) - { - var fortunes = await pool.LoadAsync( - static (id, message) => new Fortune(id, Encoding.UTF8.GetBytes(message)), - cancellationToken).ConfigureAwait(false); - await application.RenderFortunesAsync(Complete(fortunes)).ConfigureAwait(false); - } + => pool.ConsumeRetainedAsync( + static (id, message) => new Fortune(id, message), + application, + static (application, fortunes) => + application.RenderFortunesAsync(Complete(fortunes)), + cancellationToken); public override ValueTask DisposeAsync() => pool.DisposeAsync(); } diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index 2cd0400..ce915dc 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -14,8 +14,6 @@ Set all of these environment variables before starting the app: | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | -| `SLON_POOL_MODE` | `raw` (default) or `connection` | -| `SLON_CONSUMPTION_MODE` | `stream` (default) or `collect` | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -23,14 +21,11 @@ unmerged branch. ## Driver strategies -Slon uses its experimental lower layer directly in both modes, and creates a fresh -`ReaderDrivenCommandFlow` per request. `raw` opens `DATABASE_CONNECTIONS` protocols and places -flows by atomic round-robin. `connection` wraps the same protocols in `ConnectionPool` through -the lower-layer `IPoolConnection` seam, exercising production placement without adding ADO. -Every wire receives the same prepared statement before it becomes schedulable. In `stream` mode, -the response retains UTF-8 field memory through rendering, avoiding per-row strings and byte arrays. -`collect` exercises the one-await collector and materializes strings before rendering. Both Slon -pool modes disable zero-byte reads to match Apex's ordinary BCL transport shape. +Slon uses its experimental lower layer through `ConnectionPool` and creates a fresh +`ReaderDrivenCommandFlow` per request. Every wire receives the same prepared statement before it +becomes schedulable. Streaming consumption retains UTF-8 field memory through rendering, avoiding +per-row strings and byte arrays. Zero-byte reads are disabled to match Apex's ordinary BCL transport +shape. Npgsql uses a slim data source and a command bound to each leased connection. Every strategy appends and ordinally sorts the same logical model and renders through the same RazorSlices UTF-8 diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj index 0dac500..51aebfb 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj @@ -11,8 +11,7 @@ - - + From 7c5cb6a424676abacddf8250c371ca38cd49db5a Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Mon, 31 Aug 2026 17:31:42 +0200 Subject: [PATCH 133/136] Add raw Fortunes templating --- .../BenchmarkApplication.cs | 16 +++++++ .../Slon.Fortunes.Platform/Program.cs | 7 +++ .../Slon.Fortunes.Platform/README.md | 4 ++ .../RawFortuneTemplating.cs | 44 +++++++++++++++++++ .../Templates/Fortunes.cshtml | 2 +- 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 Slon.Benchmarks/Slon.Fortunes.Platform/RawFortuneTemplating.cs diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs index 794820e..91cfb5d 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/BenchmarkApplication.cs @@ -7,6 +7,12 @@ namespace Slon.Fortunes.Platform; +internal enum FortuneTemplating +{ + Razor, + Raw +} + public sealed partial class BenchmarkApplication { private static readonly DefaultObjectPool ChunkedWriterPool = @@ -15,6 +21,7 @@ public sealed partial class BenchmarkApplication private RequestType _requestType; internal static FortuneDatabase Database { get; set; } = null!; + internal static FortuneTemplating Templating { get; set; } public void OnStartLine( HttpVersionAndMethod versionAndMethod, @@ -36,6 +43,15 @@ public void OnStartLine( internal ValueTask RenderFortunesAsync(List fortunes) { + if (Templating is FortuneTemplating.Raw) + { + var writer = StartResponse(Writer); + RawFortuneTemplating.Render(fortunes, writer, HtmlEncoder); + writer.Complete(); + ReturnChunkedWriter(writer); + return ValueTask.CompletedTask; + } + var template = Templates.Fortunes.Create(fortunes); return OutputFortunesAsync(Writer, template); } diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs index d611964..a115633 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Program.cs @@ -10,6 +10,13 @@ .AddCommandLine(args) .Build(); var url = new Uri(configuration["urls"] ?? "http://0.0.0.0:5000"); +BenchmarkApplication.Templating = configuration["TEMPLATING"]?.Trim().ToLowerInvariant() switch +{ + null or "" or "razor" => FortuneTemplating.Razor, + "raw" => FortuneTemplating.Raw, + var value => throw new ArgumentOutOfRangeException( + "TEMPLATING", value, "Expected 'razor' or 'raw'."), +}; await using var database = await FortuneDatabase.CreateAsync( configuration["DATABASE"], diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index ce915dc..88e947c 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -14,6 +14,7 @@ Set all of these environment variables before starting the app: | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | +| `TEMPLATING` | `razor` (default) or `raw` | Invalid, unsupported, or missing selections fail application startup with an explicit error. The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an @@ -27,6 +28,9 @@ becomes schedulable. Streaming consumption retains UTF-8 field memory through re per-row strings and byte arrays. Zero-byte reads are disabled to match Apex's ordinary BCL transport shape. +`TEMPLATING=raw` writes the same encoded HTML directly into the response buffer. It isolates the +driver and pool cost from RazorSlices overhead without changing query or row-consumption behavior. + Npgsql uses a slim data source and a command bound to each leased connection. Every strategy appends and ordinally sorts the same logical model and renders through the same RazorSlices UTF-8 template. diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/RawFortuneTemplating.cs b/Slon.Benchmarks/Slon.Fortunes.Platform/RawFortuneTemplating.cs new file mode 100644 index 0000000..6f6fed8 --- /dev/null +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/RawFortuneTemplating.cs @@ -0,0 +1,44 @@ +using System.Buffers; +using System.Buffers.Text; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text.Encodings.Web; + +namespace Slon.Fortunes.Platform; + +internal static class RawFortuneTemplating +{ + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + internal static void Render( + List fortunes, IBufferWriter writer, HtmlEncoder encoder) + { + const int FortunesTemplateLength = 1232; + var span = writer.GetSpan(FortunesTemplateLength); + var startingLength = span.Length; + span = span.WriteAndSlice( + "Fortunes"u8); + foreach (var fortune in fortunes) + { + var current = span.WriteAndSlice(""u8); + } + span = span.WriteAndSlice("
idmessage
"u8); + var success = Utf8Formatter.TryFormat((uint)fortune.Id, current, out var written); + Debug.Assert(success); + current = current.Slice(written).WriteAndSlice(""u8); + var status = encoder.EncodeUtf8( + fortune.Message.Span, current, out _, out written, isFinalBlock: true); + Debug.Assert(status is OperationStatus.Done); + span = current.Slice(written).WriteAndSlice("
"u8); + writer.Advance(startingLength - span.Length); + } +} + +internal static class SpanExtensions +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Span WriteAndSlice(this Span destination, ReadOnlySpan source) + { + source.CopyTo(destination); + return destination.Slice(source.Length); + } +} diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml index b36d18a..1cbd846 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/Templates/Fortunes.cshtml @@ -1,2 +1,2 @@ @inherits RazorSlice> -Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message.Span
+Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message.Span
\ No newline at end of file From a4fc50cec47fa0c849598a7d6967ef8c7b700d29 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Fri, 4 Sep 2026 16:52:18 +0200 Subject: [PATCH 134/136] Run Slon fortunes through command result collection --- Slon.Benchmarks/Shared/SlonConnectionPool.cs | 43 ++++++++++++------- .../Slon.Fortunes.Minimal/README.md | 6 +-- .../Slon.Fortunes.Platform/README.md | 8 ++-- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/Slon.Benchmarks/Shared/SlonConnectionPool.cs b/Slon.Benchmarks/Shared/SlonConnectionPool.cs index e34c1d0..1c4455a 100644 --- a/Slon.Benchmarks/Shared/SlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/SlonConnectionPool.cs @@ -14,10 +14,10 @@ internal sealed class SlonConnectionPool : IAsyncDisposable { const string Query = "SELECT id, message FROM fortune"; readonly ConnectionPool _pool; - readonly ReaderDrivenCommandOptions _options; + readonly CommandFlowOptions _options; SlonConnectionPool(ConnectionPool pool, Command command) - => (_pool, _options) = (pool, new ReaderDrivenCommandOptions(command)); + => (_pool, _options) = (pool, new() { Commands = new(command) }); internal static async ValueTask CreateAsync( string connectionString, @@ -63,7 +63,7 @@ public async ValueTask> LoadAsync( Func create, CancellationToken cancellationToken) { - var flow = new ReaderDrivenCommandFlow(_options); + var flow = new CommandFlow(async: true, _options); await _pool.GetAsync( static (candidate, item) => candidate.Connection.Protocol.TryQueue( item, @@ -76,10 +76,23 @@ await _pool.GetAsync( cancellationToken).ConfigureAwait(false); var values = new List(); - await foreach (var result in flow.GetAsyncEnumerator(cancellationToken)) - await foreach (var row in result) - values.Add(create(row.GetValue(0), row.GetValue(1))); - return values; + var results = flow.GetAsyncEnumerator(cancellationToken); + try + { + while (await results.MoveNextAsync().ConfigureAwait(false)) + { + await results.Current.CollectAsync( + (Values: values, Create: create), + static (state, row) => state.Values.Add( + state.Create(row.GetInt32(0), row.GetValue(1))), + cancellationToken).ConfigureAwait(false); + } + return values; + } + finally + { + await results.DisposeAsync().ConfigureAwait(false); + } } public async ValueTask ConsumeRetainedAsync( @@ -88,7 +101,7 @@ public async ValueTask ConsumeRetainedAsync( Func, ValueTask> consume, CancellationToken cancellationToken) { - var flow = new ReaderDrivenCommandFlow(_options); + var flow = new CommandFlow(async: true, _options); await _pool.GetAsync( static (candidate, item) => candidate.Connection.Protocol.TryQueue( item, @@ -106,13 +119,13 @@ await _pool.GetAsync( { if (await results.MoveNextAsync().ConfigureAwait(false)) { - var rows = results.Current.GetAsyncEnumerator(); - while (await rows.MoveNextAsync().ConfigureAwait(false)) - { - var reader = rows.Current.GetReader(); - values.Add(create(reader.Read(), reader.ReadMemory())); - } - await rows.DisposeAsync().ConfigureAwait(false); + results.Current.EnableResultBuffering(); + await results.Current.CollectAsync( + (Values: values, Create: create), + static (collection, row) => collection.Values.Add( + collection.Create( + row.GetInt32(0), row.BorrowFieldMemory(1))), + cancellationToken).ConfigureAwait(false); } await consume(state, values).ConfigureAwait(false); } diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md index dd966e6..f1f7a40 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -22,9 +22,9 @@ unmerged branch. ## Driver strategies Slon uses its experimental lower layer through `ConnectionPool` and creates a fresh -`ReaderDrivenCommandFlow` per request. Every wire receives the same prepared statement before it -becomes schedulable. Results are consumed through nested streaming enumeration, and zero-byte reads -are disabled to match Apex's ordinary BCL transport shape. +`CommandFlow` per request. Every wire receives the same prepared statement before it becomes +schedulable. Results are consumed through `CommandResult.CollectAsync`, and zero-byte reads are +disabled to match Apex's ordinary BCL transport shape. Npgsql uses a slim data source and a command bound to each leased connection. Both drivers materialize messages as strings, append and ordinally sort the same model, and render the same diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index 88e947c..7341158 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -23,10 +23,10 @@ unmerged branch. ## Driver strategies Slon uses its experimental lower layer through `ConnectionPool` and creates a fresh -`ReaderDrivenCommandFlow` per request. Every wire receives the same prepared statement before it -becomes schedulable. Streaming consumption retains UTF-8 field memory through rendering, avoiding -per-row strings and byte arrays. Zero-byte reads are disabled to match Apex's ordinary BCL transport -shape. +`CommandFlow` per request. Every wire receives the same prepared statement before it becomes +schedulable. `CommandResult.CollectAsync` is the row-buffering barrier, while result buffering retains +UTF-8 field memory through rendering without per-row strings or byte arrays. Zero-byte reads are +disabled to match Apex's ordinary BCL transport shape. `TEMPLATING=raw` writes the same encoded HTML directly into the response buffer. It isolates the driver and pool cost from RazorSlices overhead without changing query or row-consumption behavior. From c65b871fb7d6b2841f3a0e7c7de360dd38532e57 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 12:33:42 +0200 Subject: [PATCH 135/136] Benchmark reusable command flows in fortunes apps --- Slon.Benchmarks/Shared/SlonConnectionPool.cs | 60 +++++++++++++++++-- .../Slon.Fortunes.Minimal/README.md | 15 +++-- .../minimal-fortunes.benchmarks.yml | 10 ++-- .../Slon.Fortunes.Platform/README.md | 15 +++-- .../platform-fortunes.benchmarks.yml | 10 ++-- Slon/Slon.csproj | 3 +- 6 files changed, 87 insertions(+), 26 deletions(-) diff --git a/Slon.Benchmarks/Shared/SlonConnectionPool.cs b/Slon.Benchmarks/Shared/SlonConnectionPool.cs index 1c4455a..581dc35 100644 --- a/Slon.Benchmarks/Shared/SlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/SlonConnectionPool.cs @@ -1,5 +1,7 @@ using System.Diagnostics; +using System.Globalization; using System.Net; +using Microsoft.Extensions.ObjectPool; using Npgsql; using Slon.Pg; using Slon.Pg.Protocol; @@ -15,9 +17,21 @@ internal sealed class SlonConnectionPool : IAsyncDisposable const string Query = "SELECT id, message FROM fortune"; readonly ConnectionPool _pool; readonly CommandFlowOptions _options; + readonly ObjectPool? _flowPool; - SlonConnectionPool(ConnectionPool pool, Command command) - => (_pool, _options) = (pool, new() { Commands = new(command) }); + SlonConnectionPool( + ConnectionPool pool, + Command command, + int flowPoolCapacity) + { + _pool = pool; + _options = new() { Commands = new(command) }; + if (flowPoolCapacity > 0) + { + _flowPool = new DefaultObjectPool( + new CommandFlowPoolPolicy(), flowPoolCapacity); + } + } internal static async ValueTask CreateAsync( string connectionString, @@ -56,14 +70,14 @@ internal static async ValueTask CreateAsync( MaxConnections = connectionCount, ConnectionIdleLifetime = Timeout.InfiniteTimeSpan, }); - return new(pool, command); + return new(pool, command, GetFlowPoolCapacity()); } public async ValueTask> LoadAsync( Func create, CancellationToken cancellationToken) { - var flow = new CommandFlow(async: true, _options); + var flow = RentFlow(); await _pool.GetAsync( static (candidate, item) => candidate.Connection.Protocol.TryQueue( item, @@ -92,6 +106,7 @@ await results.Current.CollectAsync( finally { await results.DisposeAsync().ConfigureAwait(false); + _flowPool?.Return(flow); } } @@ -101,7 +116,7 @@ public async ValueTask ConsumeRetainedAsync( Func, ValueTask> consume, CancellationToken cancellationToken) { - var flow = new CommandFlow(async: true, _options); + var flow = RentFlow(); await _pool.GetAsync( static (candidate, item) => candidate.Connection.Protocol.TryQueue( item, @@ -132,11 +147,46 @@ await results.Current.CollectAsync( finally { await results.DisposeAsync().ConfigureAwait(false); + _flowPool?.Return(flow); } } public ValueTask DisposeAsync() => _pool.DisposeAsync(); + CommandFlow RentFlow() + { + var flow = _flowPool?.Get(); + return flow is null + ? new CommandFlow(async: true, _options) + : flow.Initialize(async: true, _options); + } + + static int GetFlowPoolCapacity() + { + const string name = "SLON_FLOW_POOL_CAPACITY"; + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrWhiteSpace(value)) + return 0; + return int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var capacity) + && capacity >= 0 + ? capacity + : throw new InvalidOperationException($"{name} must be a non-negative integer."); + } + + sealed class CommandFlowPoolPolicy : PooledObjectPolicy + { + public override CommandFlow Create() + => new(async: true, ReadOnlySpan.Empty); + + public override bool Return(CommandFlow flow) + { + // DisposeAsync crosses the framework's retirement boundary before a flow reaches here. + // Reset therefore cannot overlap the old tenure's protocol or heartbeat observation. + flow.Reset(); + return true; + } + } + static async ValueTask PrepareAsync(PgClientProtocol protocol) { var command = Command.Create(Query, commandName: new EncodedCString("fortunes")); diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md index f1f7a40..b08f053 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -14,15 +14,17 @@ Set the following configuration values as environment variables or equivalent .N | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | +| `SLON_FLOW_POOL_CAPACITY` | Retained `CommandFlow` count; omitted or `0` disables flow pooling | Invalid, unsupported, or missing selections fail application startup with an explicit error. -The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an -unmerged branch. +The Crank config currently composes `sebros/slon-benchmarks` with Draghi's +`experiment/observation-frontier`; override either revision after those experiments land. ## Driver strategies -Slon uses its experimental lower layer through `ConnectionPool` and creates a fresh -`CommandFlow` per request. Every wire receives the same prepared statement before it becomes +Slon uses its experimental lower layer through `ConnectionPool`. Setting +`SLON_FLOW_POOL_CAPACITY` reuses `CommandFlow` instances after framework retirement; leaving it +unset creates a fresh flow per request. Every wire receives the same prepared statement before it becomes schedulable. Results are consumed through `CommandResult.CollectAsync`, and zero-byte reads are disabled to match Apex's ordinary BCL transport shape. @@ -30,5 +32,6 @@ Npgsql uses a slim data source and a command bound to each leased connection. Bo materialize messages as strings, append and ordinally sort the same model, and render the same RazorSlices string template for a fair comparison. -The Crank configuration uses two fewer Slon connections than database cores and 256 Npgsql -connections; Npgsql needs the additional in-flight operations to hide network and query latency. +The Crank configuration retains up to 1024 Slon flows, uses two fewer Slon connections than +database cores, and uses 256 Npgsql connections; Npgsql needs the additional in-flight operations +to hide network and query latency. Set `slonFlowPoolCapacity=0` for the unpooled comparison. diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml b/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml index 80cca1a..6aa04de 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/minimal-fortunes.benchmarks.yml @@ -5,8 +5,9 @@ imports: variables: serverPort: 5000 npgsqlConnections: 256 - # Override this value to benchmark an unmerged Slon branch or commit. - branchOrCommit: main + slonFlowPoolCapacity: 1024 + branchOrCommit: sebros/slon-benchmarks + draghiBranchOrCommit: experiment/observation-frontier jobs: minimal-postgresql-slon: @@ -16,7 +17,7 @@ jobs: branchOrCommit: "{{branchOrCommit}}" Draghi: repository: https://github.com/draghidev/pipelining.git - branchOrCommit: main + branchOrCommit: "{{draghiBranchOrCommit}}" project: Slon/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj framework: net10.0 patchReferences: true @@ -29,6 +30,7 @@ jobs: DRIVER: slon CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass" DATABASE_CONNECTIONS: "{{ cores | minus: 2 }}" + SLON_FLOW_POOL_CAPACITY: "{{slonFlowPoolCapacity}}" minimal-postgresql-npgsql: sources: @@ -37,7 +39,7 @@ jobs: branchOrCommit: "{{branchOrCommit}}" Draghi: repository: https://github.com/draghidev/pipelining.git - branchOrCommit: main + branchOrCommit: "{{draghiBranchOrCommit}}" project: Slon/Slon.Benchmarks/Slon.Fortunes.Minimal/Slon.Fortunes.Minimal.csproj framework: net10.0 patchReferences: true diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md index 7341158..f33923c 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/README.md @@ -14,16 +14,18 @@ Set all of these environment variables before starting the app: | `DRIVER` | `slon` or `npgsql` | | `CONNECTION_STRING` | PostgreSQL connection string | | `DATABASE_CONNECTIONS` | Positive fixed pool size | +| `SLON_FLOW_POOL_CAPACITY` | Retained `CommandFlow` count; omitted or `0` disables flow pooling | | `TEMPLATING` | `razor` (default) or `raw` | Invalid, unsupported, or missing selections fail application startup with an explicit error. -The Crank config defaults `branchOrCommit` to `main`; override it when benchmarking an -unmerged branch. +The Crank config currently composes `sebros/slon-benchmarks` with Draghi's +`experiment/observation-frontier`; override either revision after those experiments land. ## Driver strategies -Slon uses its experimental lower layer through `ConnectionPool` and creates a fresh -`CommandFlow` per request. Every wire receives the same prepared statement before it becomes +Slon uses its experimental lower layer through `ConnectionPool`. Setting +`SLON_FLOW_POOL_CAPACITY` reuses `CommandFlow` instances after framework retirement; leaving it +unset creates a fresh flow per request. Every wire receives the same prepared statement before it becomes schedulable. `CommandResult.CollectAsync` is the row-buffering barrier, while result buffering retains UTF-8 field memory through rendering without per-row strings or byte arrays. Zero-byte reads are disabled to match Apex's ordinary BCL transport shape. @@ -35,5 +37,6 @@ Npgsql uses a slim data source and a command bound to each leased connection. Ev appends and ordinally sorts the same logical model and renders through the same RazorSlices UTF-8 template. -The Crank configuration uses two fewer Slon connections than database cores and 256 Npgsql -connections. +The Crank configuration retains up to 1024 Slon flows, uses two fewer Slon connections than +database cores, and uses 256 Npgsql connections. Set `slonFlowPoolCapacity=0` for the unpooled +comparison. diff --git a/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml b/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml index 09ce85a..1d42bdd 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml +++ b/Slon.Benchmarks/Slon.Fortunes.Platform/platform-fortunes.benchmarks.yml @@ -5,8 +5,9 @@ imports: variables: serverPort: 5000 npgsqlConnections: 256 - # Override this value to benchmark an unmerged Slon branch or commit. - branchOrCommit: main + slonFlowPoolCapacity: 1024 + branchOrCommit: sebros/slon-benchmarks + draghiBranchOrCommit: experiment/observation-frontier jobs: platform-postgresql-slon: @@ -16,7 +17,7 @@ jobs: branchOrCommit: "{{branchOrCommit}}" Draghi: repository: https://github.com/draghidev/pipelining.git - branchOrCommit: main + branchOrCommit: "{{draghiBranchOrCommit}}" project: Slon/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj framework: net10.0 patchReferences: true @@ -29,6 +30,7 @@ jobs: DRIVER: slon CONNECTION_STRING: "Host={{databaseServer}};Database=hello_world;Username=benchmarkdbuser;Password=benchmarkdbpass" DATABASE_CONNECTIONS: "{{ cores | minus: 2 }}" + SLON_FLOW_POOL_CAPACITY: "{{slonFlowPoolCapacity}}" platform-postgresql-npgsql: sources: @@ -37,7 +39,7 @@ jobs: branchOrCommit: "{{branchOrCommit}}" Draghi: repository: https://github.com/draghidev/pipelining.git - branchOrCommit: main + branchOrCommit: "{{draghiBranchOrCommit}}" project: Slon/Slon.Benchmarks/Slon.Fortunes.Platform/Slon.Fortunes.Platform.csproj framework: net10.0 patchReferences: true diff --git a/Slon/Slon.csproj b/Slon/Slon.csproj index 083ddc8..19e1074 100644 --- a/Slon/Slon.csproj +++ b/Slon/Slon.csproj @@ -7,6 +7,7 @@ true preview true + $(MSBuildThisFileDirectory)../../Draghi $(TargetsForTfmSpecificBuildOutput);IncludeProjectReferenceDlls $(TargetsForTfmSpecificContentInPackage);IncludeReferenceAssemblies $(NoWarn);NU5131;DRAGHI001;SLONPG001;SLONPOOL001 @@ -23,7 +24,7 @@ - + From 1fdcc485bce57cd5842992aa06cf8aa5a7ebdc62 Mon Sep 17 00:00:00 2001 From: Nino Floris Date: Sat, 5 Sep 2026 12:45:09 +0200 Subject: [PATCH 136/136] Retain Slon row memory through minimal rendering --- Slon.Benchmarks/Shared/SlonConnectionPool.cs | 37 ---------------- .../Slon.Fortunes.Minimal/Fortune.cs | 6 +-- .../Slon.Fortunes.Minimal/FortuneDatabase.cs | 43 ++++++++++++++----- .../Slon.Fortunes.Minimal/Program.cs | 8 ++-- .../Slon.Fortunes.Minimal/README.md | 11 ++--- .../Templates/Fortunes.cshtml | 2 +- 6 files changed, 45 insertions(+), 62 deletions(-) diff --git a/Slon.Benchmarks/Shared/SlonConnectionPool.cs b/Slon.Benchmarks/Shared/SlonConnectionPool.cs index 581dc35..01d559b 100644 --- a/Slon.Benchmarks/Shared/SlonConnectionPool.cs +++ b/Slon.Benchmarks/Shared/SlonConnectionPool.cs @@ -73,43 +73,6 @@ internal static async ValueTask CreateAsync( return new(pool, command, GetFlowPoolCapacity()); } - public async ValueTask> LoadAsync( - Func create, - CancellationToken cancellationToken) - { - var flow = RentFlow(); - await _pool.GetAsync( - static (candidate, item) => candidate.Connection.Protocol.TryQueue( - item, - candidate.IsIdleCandidate - ? FlowEnqueueOptions.None - : FlowEnqueueOptions.RequireExistingPipeline, - candidate.CancellationToken), - flow, - Timeout.InfiniteTimeSpan, - cancellationToken).ConfigureAwait(false); - - var values = new List(); - var results = flow.GetAsyncEnumerator(cancellationToken); - try - { - while (await results.MoveNextAsync().ConfigureAwait(false)) - { - await results.Current.CollectAsync( - (Values: values, Create: create), - static (state, row) => state.Values.Add( - state.Create(row.GetInt32(0), row.GetValue(1))), - cancellationToken).ConfigureAwait(false); - } - return values; - } - finally - { - await results.DisposeAsync().ConfigureAwait(false); - _flowPool?.Return(flow); - } - } - public async ValueTask ConsumeRetainedAsync( Func, T> create, TState state, diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs index 945e3e2..ecea74a 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Fortune.cs @@ -2,7 +2,7 @@ namespace Slon.Fortunes.Minimal; public readonly struct Fortune : IComparable { - public Fortune(int id, string message) + public Fortune(int id, ReadOnlyMemory message) { Id = id; Message = message; @@ -10,8 +10,8 @@ public Fortune(int id, string message) public int Id { get; } - public string Message { get; } + public ReadOnlyMemory Message { get; } public int CompareTo(Fortune other) => - StringComparer.Ordinal.Compare(Message, other.Message); + Message.Span.SequenceCompareTo(other.Message.Span); } diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs index 2d2123b..807709d 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/FortuneDatabase.cs @@ -1,4 +1,6 @@ using System.Globalization; +using System.IO.Pipelines; +using System.Text.Encodings.Web; using Npgsql; using Slon.Fortunes; @@ -7,11 +9,14 @@ namespace Slon.Fortunes.Minimal; internal abstract class FortuneDatabase : IAsyncDisposable { protected const string Query = "SELECT id, message FROM fortune"; - private const string AdditionalFortune = "Additional fortune added at request time."; + private static readonly ReadOnlyMemory AdditionalFortune = + "Additional fortune added at request time."u8.ToArray(); public abstract ValueTask DisposeAsync(); - public abstract ValueTask> LoadAsync( + public abstract ValueTask RenderAsync( + PipeWriter writer, + HtmlEncoder htmlEncoder, CancellationToken cancellationToken); public static ValueTask CreateAsync(IConfiguration configuration) @@ -40,6 +45,15 @@ protected static List Complete(List fortunes) return fortunes; } + protected static async ValueTask RenderFortunesAsync( + List fortunes, + PipeWriter writer, + HtmlEncoder htmlEncoder) + { + using var template = Templates.Fortunes.Create(Complete(fortunes)); + await template.RenderAsync(writer, htmlEncoder); + } + private static string RequiredDatabase(string? value) { var database = RequiredValue("DATABASE", value); @@ -86,12 +100,16 @@ public static async ValueTask CreateAsync( => new SlonFortuneDatabase(await SlonConnectionPool.CreateAsync( connectionString, connectionCount).ConfigureAwait(false)); - public override async ValueTask> LoadAsync(CancellationToken cancellationToken) - { - var fortunes = await pool.LoadAsync( - static (id, message) => new Fortune(id, message), cancellationToken).ConfigureAwait(false); - return Complete(fortunes); - } + public override ValueTask RenderAsync( + PipeWriter writer, + HtmlEncoder htmlEncoder, + CancellationToken cancellationToken) + => pool.ConsumeRetainedAsync( + static (id, message) => new Fortune(id, message), + (Writer: writer, HtmlEncoder: htmlEncoder), + static (output, fortunes) => RenderFortunesAsync( + fortunes, output.Writer, output.HtmlEncoder), + cancellationToken); public override ValueTask DisposeAsync() => pool.DisposeAsync(); } @@ -109,7 +127,9 @@ public NpgsqlFortuneDatabase(string connectionString, int connectionCount) _dataSource = new NpgsqlSlimDataSourceBuilder(builder.ConnectionString).Build(); } - public override async ValueTask> LoadAsync( + public override async ValueTask RenderAsync( + PipeWriter writer, + HtmlEncoder htmlEncoder, CancellationToken cancellationToken) { await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); @@ -118,10 +138,11 @@ public override async ValueTask> LoadAsync( List fortunes = []; while (await reader.ReadAsync(cancellationToken)) { - fortunes.Add(new Fortune(reader.GetInt32(0), reader.GetString(1))); + fortunes.Add(new Fortune( + reader.GetInt32(0), reader.GetFieldValue(1))); } - return Complete(fortunes); + await RenderFortunesAsync(fortunes, writer, htmlEncoder); } public override ValueTask DisposeAsync() => _dataSource.DisposeAsync(); diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs b/Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs index 802d4a3..e4c1b12 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Program.cs @@ -1,7 +1,6 @@ using System.Text.Encodings.Web; using System.Text.Unicode; using Slon.Fortunes.Minimal; -using Slon.Fortunes.Minimal.Templates; var builder = WebApplication.CreateBuilder(args); @@ -14,11 +13,10 @@ app.MapGet( "/fortunes", - async (HtmlEncoder htmlEncoder, CancellationToken cancellationToken) => + async (HttpResponse response, HtmlEncoder htmlEncoder, CancellationToken cancellationToken) => { - var template = Fortunes.Create(await database.LoadAsync(cancellationToken)); - template.HtmlEncoder = htmlEncoder; - return template; + response.ContentType = "text/html; charset=utf-8"; + await database.RenderAsync(response.BodyWriter, htmlEncoder, cancellationToken); }); app.Lifetime.ApplicationStarted.Register(static () => Console.WriteLine("Application started.")); diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md index b08f053..fd2b028 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/README.md @@ -25,12 +25,13 @@ The Crank config currently composes `sebros/slon-benchmarks` with Draghi's Slon uses its experimental lower layer through `ConnectionPool`. Setting `SLON_FLOW_POOL_CAPACITY` reuses `CommandFlow` instances after framework retirement; leaving it unset creates a fresh flow per request. Every wire receives the same prepared statement before it becomes -schedulable. Results are consumed through `CommandResult.CollectAsync`, and zero-byte reads are -disabled to match Apex's ordinary BCL transport shape. +schedulable. Results are consumed through `CommandResult.CollectAsync`; result retention keeps +borrowed UTF-8 fields valid through Razor rendering without per-row strings or byte arrays. Zero-byte +reads are disabled to match Apex's ordinary BCL transport shape. -Npgsql uses a slim data source and a command bound to each leased connection. Both drivers -materialize messages as strings, append and ordinally sort the same model, and render the same -RazorSlices string template for a fair comparison. +Npgsql uses a slim data source and a command bound to each leased connection. Both drivers append, +ordinally sort, and render the same UTF-8 model; Npgsql returns each field as an allocated byte array +because its reader does not retain row storage through rendering. The Crank configuration retains up to 1024 Slon flows, uses two fewer Slon connections than database cores, and uses 256 Npgsql connections; Npgsql needs the additional in-flight operations diff --git a/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml b/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml index 39a4cb9..b36d18a 100644 --- a/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml +++ b/Slon.Benchmarks/Slon.Fortunes.Minimal/Templates/Fortunes.cshtml @@ -1,2 +1,2 @@ @inherits RazorSlice> -Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message
+Fortunes@foreach (var item in Model){}
idmessage
@WriteNumber(item.Id, default, CultureInfo.InvariantCulture, false)@item.Message.Span