From 4901148d3ebea0cb85f6192ad8d450589a9836d3 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 4 Nov 2026 14:03:22 +0000 Subject: [PATCH 01/27] MOVING handoff: hold new work back while draining, so the recycle happens within a round trip instead of at the deadline The MoveTo/Recycle paths drained by waiting for "no caller work" before replacing the connections. Under steady traffic that never happens: there is always a command in flight, so the drain waited out the entire announced window and recycled at the deadline, failing whatever was in flight at that instant and leaving no time to establish the replacement. Measured live on a Redis Enterprise endpoint rebind with ~450 ops/s on the connection: MOVING at +5.8s, MoveTo decided at +6.6s, connections replaced at +20.9s against a 15s window, 8 failed commands, and the "did not establish a replacement within the announced window" warning. The bridge now has a quiesce mode for the duration of the drain: new caller messages (single, batch, sync and async) go to the backlog instead of the socket, the backlog processor does not write while quiescing, and the drain waits only for commands already written. Held work is written to the replacement once it is established, as across any reconnect; a FailFast backlog policy no longer abandons it for a MaintenanceHandoff, which is a replacement we chose rather than a failure. Internal traffic (keep-alives) is not held. Same scenario with this change: MOVING +5.8s, handoff +6.7s, restored +6.8s, 0 failed, nothing outstanding at the recycle, no warning. The new test runs a steady load against the in-process server and asserts that the recycle follows the notification within a fraction of the window and that no command fails; it fails on main (recycle at ~10s, the whole window) and passes here. --- src/StackExchange.Redis/PhysicalBridge.cs | 62 +++++++++++++++- .../ServerEndPoint.Maintenance.cs | 42 ++++++++--- src/StackExchange.Redis/ServerEndPoint.cs | 7 ++ .../MaintenanceNotificationTests.cs | 70 +++++++++++++++++++ 4 files changed, 167 insertions(+), 14 deletions(-) diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index ee3a7bfb2..c82d066bc 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -132,6 +132,42 @@ public enum State : byte /// internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); + private volatile bool _quiescing; + + /// + /// Starts holding caller work back from the current connection while a maintenance handoff lets what has + /// already been written complete; see . + /// + /// + /// Held work goes to the backlog rather than the socket, and is written to the replacement connection once + /// that is established. Without this the drain could never finish under steady traffic: there was always + /// something in flight, so the recycle waited for the end of the announced window and then failed whatever + /// was in flight at that instant. + /// + internal void BeginQuiesce() => _quiescing = true; + + internal void EndQuiesce() + { + _quiescing = false; + if (BacklogHasItems) + { + // the processor may be lingering between passes, or may have exited; either way, get the held work moving + _backlogAutoReset.Set(); + StartBacklogProcessor(); + } + } + + /// + /// Queue a caller's message for the replacement connection during a handoff, regardless of the backlog policy: + /// the connection is live and the message would have been written, so failing it would be a regression. + /// + private WriteResult HoldForHandoff(Message message) + { + message.SetEnqueued(null); + BacklogEnqueue(message); + return WriteResult.Success; + } + public void Dispose() { isDisposed = true; @@ -212,6 +248,7 @@ private WriteResult FailDueToNoConnection(Message message) public WriteResult TryWriteSync(Message message, bool isReplica) { if (isDisposed) throw new ObjectDisposedException(Name); + if (_quiescing && !message.IsInternalCall) return HoldForHandoff(message); if (!IsConnected || NeedsReconnect) return QueueOrFailMessage(message); var physical = this.physical; @@ -236,6 +273,7 @@ public WriteResult TryWriteSync(Message message, bool isReplica) public ValueTask TryWriteAsync(Message message, bool isReplica, bool bypassBacklog = false) { if (isDisposed) throw new ObjectDisposedException(Name); + if (_quiescing && !bypassBacklog && !message.IsInternalCall) return new ValueTask(HoldForHandoff(message)); if ((!IsConnected || NeedsReconnect) && !bypassBacklog) return new ValueTask(QueueOrFailMessage(message)); var physical = this.physical; @@ -477,8 +515,9 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur Multiplexer.Logger?.LogErrorConnectionIssue(innerException, innerException.Message); } Trace($"OnConnectionFailed: {connection}"); - // If we're configured to, fail all pending backlogged messages - if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true) + // If we're configured to, fail all pending backlogged messages - but not for a maintenance handoff, which + // is a replacement we chose, and whose backlog is work deliberately held for the replacement connection + if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff) { AbandonPendingBacklog(innerException, connection); } @@ -776,6 +815,16 @@ internal bool TryEnqueue(List messages, bool isReplica) if (isDisposed) throw new ObjectDisposedException(Name); + if (_quiescing) + { + // a maintenance handoff is draining the connection: hold the batch for the replacement + foreach (var message in messages) + { + HoldForHandoff(message); + } + return true; + } + if (!IsConnected || NeedsReconnect) { // During reconnection (e.g. MOVED-to-same-endpoint), queue messages to the backlog @@ -1072,6 +1121,13 @@ private void StartBacklogProcessor() /// Crawls from the head of the backlog queue, consuming anything that should have timed out /// and pruning it accordingly (these messages will get timeout exceptions). /// + /// + /// Whether a caller is waiting on something that has actually been written to the current connection, as + /// opposed to queued in the backlog. This is what a handoff drains: backlogged work is held for the + /// replacement rather than abandoned, so it must not keep the old connection alive. + /// + internal bool HasCallerWorkInFlight() => physical?.HasCallerMessagesAwaitingResponse() == true; + /// /// Whether this bridge owes a *caller* anything, ignoring our own internal traffic. /// @@ -1272,7 +1328,7 @@ private void ProcessBridgeBacklog() // Timeouts are handled above, so we're exclusively into backlog items eligible to write at this point. // If we can't write them, abort and wait for the next heartbeat or activation to try this again. bool flush = false; - while (IsConnected) + while (IsConnected && !_quiescing) // not while a handoff is holding work back for the replacement { // Snapshot the connection for the whole of this message: OnDisconnected nulls the field // without waiting for the write lock, so re-reading it (as the loop guard used to) can hand diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 53308faa6..d900c3440 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -471,7 +471,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? } /// - /// Lets in-flight caller work finish, then replaces the connections. + /// Holds new caller work back, lets what has already been written finish, then replaces the connections. /// /// /// Draining first because the socket is still working: anything already written may still be answered, and @@ -479,6 +479,14 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? /// because the server closes the socket at the end of it regardless - so anything not drained by then was /// going to fail either way, and draining strictly dominates. /// + /// Holding new work back is what makes the drain finish. Measured live on a connection carrying about 450 + /// operations a second: there was always something in flight, so a drain that waited for "no caller work" + /// waited out the entire 15s window and then recycled at the deadline - failing the eight commands in flight + /// at that instant, with no time left to establish the replacement. That is the exact outcome the handoff + /// exists to avoid. With new work held in the backlog, the written commands complete within a round trip, the + /// recycle follows, and the held work is written to the replacement once it is established. + /// + /// /// Both bridges, not just the one that was told. The measured blast radius is the *node*: four connections /// to one proxy, differing only in handshake, all closed simultaneously, and only the ones that had opted in /// were warned. @@ -487,18 +495,30 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) { var watch = ValueStopwatch.StartNew(); - while (HasCallerWork() && watch.ElapsedMilliseconds < budget.TotalMilliseconds) + interactive?.BeginQuiesce(); + subscription?.BeginQuiesce(); + try { - await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); - } + while (HasCallerWorkInFlight() && watch.ElapsedMilliseconds < budget.TotalMilliseconds) + { + await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); + } - var drained = !HasCallerWork(); - var recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); - if (recycled) Interlocked.Increment(ref _handoffRecycles); - Multiplexer.Trace( - $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" - + (drained ? " (drained)" : " (still busy; the window ran out)"), - ToString()); + var drained = !HasCallerWorkInFlight(); + var recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); + if (recycled) Interlocked.Increment(ref _handoffRecycles); + Multiplexer.Trace( + $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" + + (drained ? " (drained)" : " (still busy; the window ran out)"), + ToString()); + } + finally + { + // the recycle has already nulled the old connection, so releasing the hold cannot write to it; the + // held work waits in the backlog for the replacement, as it would across any reconnect + interactive?.EndQuiesce(); + subscription?.EndQuiesce(); + } } /// diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index e203139c6..f7da7a062 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -392,6 +392,13 @@ internal bool IsIdle() internal bool HasCallerWork() => interactive?.HasCallerWork() == true || subscription?.HasCallerWork() == true; + /// + /// Whether a caller is waiting on something already written to a current connection; backlogged work does + /// not count. See . + /// + internal bool HasCallerWorkInFlight() + => interactive?.HasCallerWorkInFlight() == true || subscription?.HasCallerWorkInFlight() == true; + /// /// Work this server still owes an answer on: written-and-awaiting-response, plus anything queued in /// the backlog. Zero means a retirement can complete without abandoning anyone. diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index bfdb3a7a5..a14a4ede8 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using StackExchange.Redis.Maintenance; @@ -842,6 +843,75 @@ await Poll.UntilAsync( } } + [Fact] + public async Task MovingUnderSteadyLoadRecyclesPromptlyAndFailsNothing() + { + // The drain used to wait for "no caller work", which under steady traffic never comes: measured live at + // about 450 ops/s, the recycle arrived at the end of the 15s window and failed the eight commands in + // flight at that instant, with no time left to establish the replacement. Holding new work back lets the + // written commands finish, so the recycle follows within a round trip and nothing fails: the held work is + // written to the replacement. + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); + using (server) + await using (conn) + { + // every reply is held briefly, so with several workers something is always in flight + server.SetLatency(TimeSpan.FromMilliseconds(40)); + var db = conn.GetDatabase(); + using var stop = new CancellationTokenSource(); + long completed = 0; + var failures = new List(); + var workers = Enumerable.Range(0, 4).Select(_ => Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + try + { + await db.PingAsync(); + Interlocked.Increment(ref completed); + } + catch (Exception ex) + { + lock (failures) failures.Add(ex); + } + } + })).ToArray(); + Assert.True(await Poll.UntilAsync(() => Volatile.Read(ref completed) > 20), "the load should be flowing before the handoff"); + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + var watch = System.Diagnostics.Stopwatch.StartNew(); + server.SendMoving(null, timeSeconds: 10, newEndpoint: successor, sequenceId: 0); + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + Assert.True( + await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8_000), + "the handoff should have replaced the connection"); + var recycledAfter = watch.Elapsed; + log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms of a 10s window"); + + // jitter is at most a tenth of the window and the drain is one round trip; recycling at the deadline is the bug + Assert.True(recycledAfter < TimeSpan.FromSeconds(4), $"recycled after {recycledAfter}: that is waiting for the window, not draining"); + + // the replacement carries the held work and whatever follows + var before = Volatile.Read(ref completed); + Assert.True( + await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutMilliseconds: 10_000), + "traffic should resume on the replacement connection"); + + stop.Cancel(); + await Task.WhenAll(workers); + lock (failures) + { + log.WriteLine($"completed: {Volatile.Read(ref completed)}; failed: {failures.Count}"); + Assert.Empty(failures.Select(f => f.Message)); + } + Assert.DoesNotContain(logs.Lines, l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); + } + } + [Fact] public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() { From bc523608db46fba61c8b332618f993dfae7ffec5 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 4 Nov 2026 15:38:29 +0000 Subject: [PATCH 02/27] Handoff drain: cap the wait, recycle under the write lock, hold only live caller work Review follow-ups on the quiesce-then-drain change: - The drain is capped at one second as well as by the window. A blocking or stuck command in flight would otherwise hold everyone else's work in the backlog for the rest of the window and still be failed at the end of it; past the cap the connections are replaced regardless, and whatever was still in flight fails as the server's close would have failed it. - The recycle takes the bridge's write lock and re-checks for in-flight work under it, so a writer that passed the hold check before it was set cannot slip onto the old connection between the check and the recycle. A writer that reaches the lock after a handoff recycled its connection queues for the replacement instead of writing to a dead connection; only for the handoff (an ordinary closure is still reported as one, #3167), never for a message bypassing the backlog, and never for internal traffic. - Held work goes through QueueOrFailMessage's queue branch, so QUIT is never queued and the duplication is gone; QUIT and internal traffic are not held at all, in the batch path as well as the single-message paths. - Work is held only while the connection is still live; once it has dropped, the ordinary disconnected rules and the backlog policy apply. - A held enqueue that lands just after the hold is released kicks the backlog processor itself rather than waiting for the heartbeat. - The orphaned summary is back on CheckBacklogForTimeouts, and docs/ServerMaintenanceEvent.md describes the hold. New test: slow commands in flight do not hold the drain past the cap, and traffic resumes on the replacement. --- docs/ServerMaintenanceEvent.md | 4 +- src/StackExchange.Redis/PhysicalBridge.cs | 108 +++++++++++++----- .../ServerEndPoint.Maintenance.cs | 61 ++++++++-- .../MaintenanceNotificationTests.cs | 57 +++++++++ 4 files changed, 194 insertions(+), 36 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 15ef44c5a..530465c32 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -111,10 +111,12 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did | `MIGRATING`, `FAILING_OVER`, `SMIGRATING` | relaxes command timeouts for that server while the disruption lasts | | `MIGRATED`, `FAILED_OVER` | ends the window, but keeps timeouts relaxed for a short tail while things settle | | `SMIGRATED` | as above, and re-reads the cluster topology, and re-subscribes any sharded channels whose slots moved | -| `MOVING` | works out the replacement address, lets in-flight work finish, then replaces the connections | +| `MOVING` | works out the replacement address, holds new commands back for up to a second while in-flight ones finish, then replaces the connections and sends the held commands on the replacement | So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog, for up to about a second, and are sent on the replacement once it is established. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. A command still in flight when the second is up, such as a blocking read, is failed by the replacement, as the server's close would have failed it; the hold is bounded so that one such command cannot stall everything else for the rest of the window. + ### Notifications that arrive as you connect Redis Enterprise **retains the most recent completion** - `MIGRATED` or `FAILED_OVER` - and replays it to each connection that opts in, so a client that connects after a disruption still learns that it happened. Measured behaviour, worth knowing if you handle these events yourself: diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index c82d066bc..ee752a780 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -149,23 +149,49 @@ public enum State : byte internal void EndQuiesce() { _quiescing = false; - if (BacklogHasItems) - { - // the processor may be lingering between passes, or may have exited; either way, get the held work moving - _backlogAutoReset.Set(); - StartBacklogProcessor(); - } + if (BacklogHasItems) KickBacklogProcessor(); } + /// Gets held or queued work moving: the processor may be lingering between passes, or may have exited. + private void KickBacklogProcessor() + { + _backlogAutoReset.Set(); + StartBacklogProcessor(); + } + + /// + /// Whether a message should be held in the backlog for the replacement connection rather than written: only + /// while a handoff is draining a connection that is still live (once it has dropped, the ordinary disconnected + /// rules and the backlog policy apply), and only for a caller's work. Internal traffic (keep-alives, probes) + /// still goes to the old connection, and so does QUIT, which is never queued. + /// + private bool ShouldHoldForHandoff(Message message) + => _quiescing && IsConnected && !NeedsReconnect && !message.IsInternalCall && message.Command != RedisCommand.QUIT; + /// - /// Queue a caller's message for the replacement connection during a handoff, regardless of the backlog policy: - /// the connection is live and the message would have been written, so failing it would be a regression. + /// Replaces the connection once nothing a caller is waiting on is in flight, under the write lock: a writer that + /// passed the hold check before it was set, but has not yet written, cannot slip onto the old connection between + /// the check and the recycle. Such a writer finds the connection gone when it gets the lock, and queues for the + /// replacement instead (see ). /// - private WriteResult HoldForHandoff(Message message) + /// null if caller work was still in flight, so the drain should keep waiting; otherwise whether + /// there was a connection to recycle. + internal async ValueTask TryRecycleWhenDrainedAsync(string reason) { - message.SetEnqueued(null); - BacklogEnqueue(message); - return WriteResult.Success; + if (!await _singleWriter.TryTakeAsync().ForAwait()) + { + // could not get the lock inside the write timeout: something is badly stuck, and the window is finite + return RecycleConnection(reason); + } + try + { + if (HasCallerWorkInFlight()) return null; + return RecycleConnection(reason); + } + finally + { + _singleWriter.Release(); + } } public void Dispose() @@ -212,18 +238,21 @@ public void ReportNextFailure() public override string ToString() => ConnectionType + "/" + Format.ToString(ServerEndPoint.EndPoint); - private WriteResult QueueOrFailMessage(Message message) + private WriteResult QueueOrFailMessage(Message message, bool holdForHandoff = false) { - // If it's an internal call that's not a QUIT - // or we're allowed to queue in general, then queue - if (message.IsInternalCall || Multiplexer.RawConfig.BacklogPolicy.QueueWhileDisconnected) + // If it's an internal call that's not a QUIT, or we're allowed to queue in general, then queue. Work held + // for a maintenance handoff is queued whatever the policy: the connection is live and the message would + // otherwise have been written, so failing it would be a regression (see ShouldHoldForHandoff). + if (holdForHandoff || message.IsInternalCall || Multiplexer.RawConfig.BacklogPolicy.QueueWhileDisconnected) { // Let's just never ever queue a QUIT message if (message.Command != RedisCommand.QUIT) { message.SetEnqueued(null); BacklogEnqueue(message); - // Note: we don't start a worker on each message here + // Note: we don't start a worker on each message here - except for held work that lands just after + // the hold was released, which nothing else would pick up until the heartbeat + if (holdForHandoff && !_quiescing) KickBacklogProcessor(); return WriteResult.Success; // Successfully queued, so indicate success } } @@ -248,7 +277,7 @@ private WriteResult FailDueToNoConnection(Message message) public WriteResult TryWriteSync(Message message, bool isReplica) { if (isDisposed) throw new ObjectDisposedException(Name); - if (_quiescing && !message.IsInternalCall) return HoldForHandoff(message); + if (ShouldHoldForHandoff(message)) return QueueOrFailMessage(message, holdForHandoff: true); if (!IsConnected || NeedsReconnect) return QueueOrFailMessage(message); var physical = this.physical; @@ -273,7 +302,7 @@ public WriteResult TryWriteSync(Message message, bool isReplica) public ValueTask TryWriteAsync(Message message, bool isReplica, bool bypassBacklog = false) { if (isDisposed) throw new ObjectDisposedException(Name); - if (_quiescing && !bypassBacklog && !message.IsInternalCall) return new ValueTask(HoldForHandoff(message)); + if (!bypassBacklog && ShouldHoldForHandoff(message)) return new ValueTask(QueueOrFailMessage(message, holdForHandoff: true)); if ((!IsConnected || NeedsReconnect) && !bypassBacklog) return new ValueTask(QueueOrFailMessage(message)); var physical = this.physical; @@ -602,6 +631,7 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) { reportNextFailure = reconfigureNextFailure = true; LastException = null; + _recycledForHandoff = null; Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -815,12 +845,13 @@ internal bool TryEnqueue(List messages, bool isReplica) if (isDisposed) throw new ObjectDisposedException(Name); - if (_quiescing) + if (_quiescing && IsConnected && !NeedsReconnect && messages.Exists(m => !m.IsInternalCall && m.Command != RedisCommand.QUIT)) { - // a maintenance handoff is draining the connection: hold the batch for the replacement + // a maintenance handoff is draining the connection: hold the batch for the replacement. A batch is + // written as a unit, so it is held as a unit; a batch of purely internal traffic is not held at all foreach (var message in messages) { - HoldForHandoff(message); + QueueOrFailMessage(message, holdForHandoff: true); } return true; } @@ -977,6 +1008,7 @@ internal WriteResult WriteMessageTakingWriteLockSync(PhysicalConnection physical if (!gotLock) return TimedOutBeforeWrite(message); } + if (ConnectionReplacedWhileWaiting(physical, message)) return QueueOrFailMessage(message); var result = WriteMessageInsideLock(physical, message); if ((result == WriteResult.Success) & message.IsFlushRequiredSync) @@ -1117,10 +1149,6 @@ private void StartBacklogProcessor() } } - /// - /// Crawls from the head of the backlog queue, consuming anything that should have timed out - /// and pruning it accordingly (these messages will get timeout exceptions). - /// /// /// Whether a caller is waiting on something that has actually been written to the current connection, as /// opposed to queued in the backlog. This is what a handoff drains: backlogged work is held for the @@ -1142,6 +1170,10 @@ internal bool HasCallerWork() return false; } + /// + /// Crawls from the head of the backlog queue, consuming anything that should have timed out + /// and pruning it accordingly (these messages will get timeout exceptions). + /// private void CheckBacklogForTimeouts() { var now = Environment.TickCount; @@ -1446,6 +1478,22 @@ private WriteResult TimedOutBeforeWrite(Message message) return WriteResult.TimeoutBeforeWrite; } + /// + /// The connection most recently replaced by a maintenance handoff, so that a writer which was headed for it + /// can tell a planned replacement from an ordinary closure. + /// + private PhysicalConnection? _recycledForHandoff; + + /// + /// Whether the connection a caller's message was headed for was replaced by a maintenance handoff while the + /// writer waited for the lock. The message has not been written, so it is queued for the replacement rather + /// than written to a connection that is gone. Only the handoff case: a write that loses its connection to an + /// ordinary closure is still reported as a closure (#3167), and a message bypassing the backlog is never + /// queued. Internal traffic is exempt as well: the handshake writes to a connection before it is current. + /// + private bool ConnectionReplacedWhileWaiting(PhysicalConnection physical, Message message, bool bypassBacklog = false) + => !bypassBacklog && !message.IsInternalCall && !ReferenceEquals(physical, this.physical) && ReferenceEquals(physical, _recycledForHandoff); + /// /// This writes a message to the output stream. /// @@ -1483,11 +1531,12 @@ internal ValueTask WriteMessageTakingWriteLockAsync(PhysicalConnect // if we *still* can't get it: that counts as // an actual timeout var pending = _singleWriter.TryTakeAsync(); - if (!pending.IsCompletedSuccessfully) return WriteMessageTakingWriteLockAsync_Awaited(pending, physical, message); + if (!pending.IsCompletedSuccessfully) return WriteMessageTakingWriteLockAsync_Awaited(pending, physical, message, bypassBacklog); gotLock = pending.GetAwaiter().GetResult(); // fine since we know we got a result if (!gotLock) return new ValueTask(TimedOutBeforeWrite(message)); } + if (ConnectionReplacedWhileWaiting(physical, message, bypassBacklog)) return new ValueTask(QueueOrFailMessage(message)); var result = WriteMessageInsideLock(physical, message); if ((result == WriteResult.Success) & message.IsFlushRequiredAsync) { @@ -1519,13 +1568,15 @@ internal ValueTask WriteMessageTakingWriteLockAsync(PhysicalConnect private async ValueTask WriteMessageTakingWriteLockAsync_Awaited( ValueTask pending, PhysicalConnection physical, - Message message) + Message message, + bool bypassBacklog) { bool gotLock = false; try { gotLock = await pending.ForAwait(); if (!gotLock) return TimedOutBeforeWrite(message); + if (ConnectionReplacedWhileWaiting(physical, message, bypassBacklog)) return QueueOrFailMessage(message); var result = WriteMessageInsideLock(physical, message); if (result == WriteResult.Success) @@ -1939,6 +1990,7 @@ internal bool RecycleConnection(string reason) // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first // also performs the disconnect, so the Dispose below is cleanup. + _recycledForHandoff = current; // a writer already past the hold check queues for the replacement instead current.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); current.Dispose(); return true; diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index d900c3440..b65f7893a 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -470,14 +470,19 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? } } + /// + /// How long a handoff waits for in-flight commands before replacing the connections regardless. Ordinary + /// commands clear in a round trip; anything still in flight after this is a blocking or stuck command that will + /// not finish inside the window either, and holding everyone else's work for it would be a regression. + /// + private static readonly TimeSpan DrainCap = TimeSpan.FromSeconds(1); + /// /// Holds new caller work back, lets what has already been written finish, then replaces the connections. /// /// /// Draining first because the socket is still working: anything already written may still be answered, and - /// dropping it would fail commands that were about to succeed. Bounded by what is left of the window, - /// because the server closes the socket at the end of it regardless - so anything not drained by then was - /// going to fail either way, and draining strictly dominates. + /// dropping it would fail commands that were about to succeed. /// /// Holding new work back is what makes the drain finish. Measured live on a connection carrying about 450 /// operations a second: there was always something in flight, so a drain that waited for "no caller work" @@ -487,6 +492,12 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? /// recycle follows, and the held work is written to the replacement once it is established. /// /// + /// The wait is bounded by as well as by the window: a blocking read in flight would + /// otherwise hold every other caller's work for the whole window and still be failed at the end of it. Past + /// the cap the connections are replaced regardless, and whatever was still in flight fails as the server's + /// close would have failed it. + /// + /// /// Both bridges, not just the one that was told. The measured blast radius is the *node*: four connections /// to one proxy, differing only in handshake, all closed simultaneously, and only the ones that had opted in /// were warned. @@ -495,21 +506,34 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) { var watch = ValueStopwatch.StartNew(); + var cap = budget < DrainCap ? budget : DrainCap; interactive?.BeginQuiesce(); subscription?.BeginQuiesce(); try { - while (HasCallerWorkInFlight() && watch.ElapsedMilliseconds < budget.TotalMilliseconds) + bool drained, recycled; + while (true) { + var attempt = await TryRecycleWhenDrainedAsync(reason).ForAwait(); + if (attempt is { } done) + { + drained = true; + recycled = done; + break; + } + if (watch.ElapsedMilliseconds >= cap.TotalMilliseconds) + { + drained = false; + recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); + break; + } await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } - var drained = !HasCallerWorkInFlight(); - var recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); if (recycled) Interlocked.Increment(ref _handoffRecycles); Multiplexer.Trace( $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" - + (drained ? " (drained)" : " (still busy; the window ran out)"), + + (drained ? " (drained)" : " (still busy at the drain cap)"), ToString()); } finally @@ -521,6 +545,29 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) } } + /// + /// Each bridge in turn, under its write lock: null as soon as one still has caller work in flight, + /// otherwise whether anything was recycled. A bridge recycled on an earlier pass has nothing left to recycle, + /// so a partial pass is safe to repeat. + /// + private async ValueTask TryRecycleWhenDrainedAsync(string reason) + { + bool recycled = false; + if (interactive is { } i) + { + var result = await i.TryRecycleWhenDrainedAsync(reason).ForAwait(); + if (result is null) return null; + recycled |= result.Value; + } + if (subscription is { } s) + { + var result = await s.TryRecycleWhenDrainedAsync(reason).ForAwait(); + if (result is null) return null; + recycled |= result.Value; + } + return recycled; + } + /// /// Warns when a replacement connection is not fully established by the time the announced window ends. /// diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index a14a4ede8..1c8fe4319 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -912,6 +912,63 @@ await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutM } } + [Fact] + public async Task MovingWithSlowCommandsInFlightRecyclesAtTheDrainCap() + { + // A blocking or very slow command must not hold everyone else's work in the backlog for the whole window: + // the drain waits a bounded time for in-flight commands and then recycles regardless. The slow commands are + // failed by the recycle, as the server's close would have failed them; everything issued afterwards flows. + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); + using (server) + await using (conn) + { + server.SetLatency(TimeSpan.FromSeconds(3)); // every command now outlasts the drain cap + var db = conn.GetDatabase(); + var slow = Enumerable.Range(0, 3).Select(_ => db.PingAsync()).ToArray(); + await Task.Delay(250); // let them be written + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + var watch = System.Diagnostics.Stopwatch.StartNew(); + server.SendMoving(null, timeSeconds: 10, newEndpoint: successor, sequenceId: 0); + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + Assert.True( + await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8_000), + "the handoff should have replaced the connection"); + var recycledAfter = watch.Elapsed; + server.SetLatency(TimeSpan.Zero); // so the replacement's handshake is not slowed too + log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with slow commands in flight"); + + // jitter is at most a tenth of the window, the cap one second: well short of the 3s the commands need + Assert.True(recycledAfter < TimeSpan.FromSeconds(3), $"recycled after {recycledAfter}: the drain waited for the slow commands instead of capping"); + + // the in-flight commands were failed by the recycle, which is what the server's close would have done + foreach (var pending in slow) + { + await Assert.ThrowsAnyAsync(async () => await pending); + } + + Assert.True( + await Poll.UntilAsync( + () => + { + try + { + return conn.IsConnected && conn.GetDatabase().Ping() >= TimeSpan.Zero; + } + catch (Exception ex) when (ex is RedisException or TimeoutException) + { + return false; + } + }, + timeoutMilliseconds: 30_000), + "the client should be serving commands on the replacement connection"); + } + } + [Fact] public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() { From 9a0f4abde639cce8dd2d59471378be575015a0c8 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 4 Nov 2026 16:15:07 +0000 Subject: [PATCH 03/27] Handoff drain: retire only the connection the hold began on, under the lock, with a bounded wait; requeue raced writers at the head Second round of review follow-ups: - Each bridge records the connection current when the hold begins and only ever retires that instance (TryRetireQuiesceTargetAsync). A replacement established meanwhile is never touched however often the drain loop retries, which the previous pass could do while the other bridge was still draining; and if the target went away by itself there is nothing to retire. - Every retirement happens under the write lock: the forced pass at the cap as well. The lock wait is bounded by a cancellation token the drain owns (the cap plus a second of grace) rather than the mutex timeout; if the lock cannot be had even then, the connection is left to the server's close rather than recycled around a writer. - A writer that lost its connection to the handoff while waiting for the lock is requeued as held work (never failed under FailFast) at the head of the backlog, via a front queue every backlog operation drains first, so it keeps its place ahead of what the hold queued after it. - The cap is half the relaxed timeout (five seconds by default) bounded by the window, and timeouts are relaxed for the hold, so a blocking command with a timeout up to the cap completes and nothing held is timed out by the hold itself. A fixed second failed commands the server's close would have let finish; the whole window held everyone else's work for one of them. - Each bridge is released in its own try/finally and the processor wake is guarded, so a bridge disposed mid-handoff cannot leave the other quiesced. - The two unused in-flight predicates are gone; the requeue check is one helper at the three lock sites. Tests: a slow command that fits inside the cap is answered and the recycle follows it; a longer one is failed at the cap and traffic resumes on the replacement; the steady-load test is unchanged. --- docs/ServerMaintenanceEvent.md | 4 +- src/StackExchange.Redis/PhysicalBridge.cs | 202 ++++++++++++------ .../ServerEndPoint.Maintenance.cs | 99 ++++----- src/StackExchange.Redis/ServerEndPoint.cs | 7 - .../MaintenanceNotificationTests.cs | 63 ++++-- 5 files changed, 234 insertions(+), 141 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 530465c32..7fcb8201b 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -111,11 +111,11 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did | `MIGRATING`, `FAILING_OVER`, `SMIGRATING` | relaxes command timeouts for that server while the disruption lasts | | `MIGRATED`, `FAILED_OVER` | ends the window, but keeps timeouts relaxed for a short tail while things settle | | `SMIGRATED` | as above, and re-reads the cluster topology, and re-subscribes any sharded channels whose slots moved | -| `MOVING` | works out the replacement address, holds new commands back for up to a second while in-flight ones finish, then replaces the connections and sends the held commands on the replacement | +| `MOVING` | works out the replacement address, holds new commands back while in-flight ones finish (for up to half the relaxed timeout, with timeouts relaxed meanwhile), then replaces the connections and sends the held commands on the replacement | So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog, for up to about a second, and are sent on the replacement once it is established. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. A command still in flight when the second is up, such as a blocking read, is failed by the replacement, as the server's close would have failed it; the hold is bounded so that one such command cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default); timeouts are relaxed for its duration, so nothing is timed out by the hold itself, and a blocking command with a timeout up to that completes normally. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index ee752a780..9603b3958 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -34,7 +34,15 @@ internal sealed class PhysicalBridge : IDisposable /// For now, simplicity in: queue it all, replay or timeout it all. /// private readonly ConcurrentQueue _backlog = new(); - private bool BacklogHasItems => !_backlog.IsEmpty; + + /// + /// Messages that go out before anything in : a writer that lost its connection to a + /// maintenance handoff while waiting for the write lock was issued before anything the hold has queued + /// since, so it is requeued here rather than appended. Every backlog operation drains this first. + /// + private readonly ConcurrentQueue _backlogFront = new(); + + private bool BacklogHasItems => !_backlogFront.IsEmpty || !_backlog.IsEmpty; private int _backlogProcessorIsRunning = 0; private int _backlogCurrentEnqueued = 0; private long _backlogTotalEnqueued = 0; @@ -133,10 +141,14 @@ public enum State : byte internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); private volatile bool _quiescing; + private PhysicalConnection? _quiesceTarget; + private volatile bool _quiesceRecycled; /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has - /// already been written complete; see . + /// already been written complete; see . The connection + /// current at this moment is the one the handoff will retire, and the only one it will ever retire: a + /// replacement established meanwhile is left alone. /// /// /// Held work goes to the backlog rather than the socket, and is written to the replacement connection once @@ -144,18 +156,31 @@ public enum State : byte /// something in flight, so the recycle waited for the end of the announced window and then failed whatever /// was in flight at that instant. /// - internal void BeginQuiesce() => _quiescing = true; + internal void BeginQuiesce() + { + _quiesceTarget = physical; + _quiesceRecycled = false; + _quiescing = true; + } internal void EndQuiesce() { _quiescing = false; + _quiesceTarget = null; if (BacklogHasItems) KickBacklogProcessor(); } /// Gets held or queued work moving: the processor may be lingering between passes, or may have exited. private void KickBacklogProcessor() { - _backlogAutoReset.Set(); + try + { + _backlogAutoReset.Set(); + } + catch (ObjectDisposedException) + { + return; // a bridge disposed mid-handoff has no processor to wake, and nothing left to write + } StartBacklogProcessor(); } @@ -169,24 +194,58 @@ private bool ShouldHoldForHandoff(Message message) => _quiescing && IsConnected && !NeedsReconnect && !message.IsInternalCall && message.Command != RedisCommand.QUIT; /// - /// Replaces the connection once nothing a caller is waiting on is in flight, under the write lock: a writer that - /// passed the hold check before it was set, but has not yet written, cannot slip onto the old connection between - /// the check and the recycle. Such a writer finds the connection gone when it gets the lock, and queues for the - /// replacement instead (see ). + /// Retires the connection the hold began on, under the write lock, once nothing a caller is waiting on is in + /// flight on it - or regardless, when says the drain has waited long enough. Only + /// that connection: one that replaced it meanwhile is never touched, however often this is called. The lock + /// is what keeps a writer that passed the hold check before it was set from slipping onto the old connection + /// between the check and the recycle; such a writer finds the connection replaced when it gets the lock, and + /// queues for the replacement instead (see ). /// - /// null if caller work was still in flight, so the drain should keep waiting; otherwise whether - /// there was a connection to recycle. - internal async ValueTask TryRecycleWhenDrainedAsync(string reason) + /// Replace the connection even if caller work is still in flight on it. + /// Why; for the trace. + /// Bounds the wait for the write lock: the drain decides how long, not the mutex timeout. + /// null if the connection could not be retired yet - caller work in flight, or the lock not + /// obtained inside the deadline - so the drain should try again; otherwise whether this call, or an earlier + /// one, replaced it. + internal async ValueTask TryRetireQuiesceTargetAsync(bool force, string reason, CancellationToken lockDeadline) { - if (!await _singleWriter.TryTakeAsync().ForAwait()) + var target = _quiesceTarget; + if (target is null) return false; // nothing was connected when the hold began + if (_quiesceRecycled) return true; + + bool gotLock; + try + { + gotLock = await _singleWriter.TryTakeAsync(lockDeadline).ForAwait(); + } + catch (OperationCanceledException) { - // could not get the lock inside the write timeout: something is badly stuck, and the window is finite - return RecycleConnection(reason); + gotLock = false; } + if (!gotLock) return null; try { - if (HasCallerWorkInFlight()) return null; - return RecycleConnection(reason); + if (_quiesceRecycled) return true; + if (!ReferenceEquals(physical, target)) + { + // it went away by itself (the server closed it, or it faulted) and the ordinary reconnect owns + // what happens next; there is nothing left for the handoff to retire + _quiesceRecycled = true; + return false; + } + if (!force && target.HasCallerMessagesAwaitingResponse()) return null; + + Trace($"recycling {ConnectionType} connection: {reason}"); + + // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises + // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and + // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first + // also performs the disconnect, so the Dispose below is cleanup. + _recycledForHandoff = target; // a writer already past the hold check queues for the replacement instead + _quiesceRecycled = true; + target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + target.Dispose(); + return true; } finally { @@ -416,7 +475,7 @@ public override string ToString() => MessagesSinceLastHeartbeat = (int)(Volatile.Read(ref operationCount) - Volatile.Read(ref profileLastLog)), ConnectedAt = ConnectedAt, IsWriterActive = !_singleWriter.IsAvailable, - BacklogMessagesPending = _backlog.Count, + BacklogMessagesPending = _backlogFront.Count + _backlog.Count, BacklogMessagesPendingCounter = Volatile.Read(ref _backlogCurrentEnqueued), BacklogStatus = _backlogStatus, TotalBacklogMessagesQueued = _backlogTotalEnqueued, @@ -1008,7 +1067,7 @@ internal WriteResult WriteMessageTakingWriteLockSync(PhysicalConnection physical if (!gotLock) return TimedOutBeforeWrite(message); } - if (ConnectionReplacedWhileWaiting(physical, message)) return QueueOrFailMessage(message); + if (TryRequeueForHandoff(physical, message, bypassBacklog: false, out var requeued)) return requeued; var result = WriteMessageInsideLock(physical, message); if ((result == WriteResult.Success) & message.IsFlushRequiredSync) @@ -1074,7 +1133,7 @@ internal void AcceptRerouted(Message message) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void BacklogEnqueue(Message message) { - bool wasEmpty = _backlog.IsEmpty; + bool wasEmpty = !BacklogHasItems; // important that this *precedes* enqueue, to play well with HasPendingCallerFacingItems Interlocked.Increment(ref _backlogCurrentEnqueued); Interlocked.Increment(ref _backlogTotalEnqueued); @@ -1095,7 +1154,7 @@ private void BacklogEnqueue(Message message) [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool BacklogTryDequeue([NotNullWhen(true)] out Message? message) { - if (_backlog.TryDequeue(out message)) + if (_backlogFront.TryDequeue(out message) || _backlog.TryDequeue(out message)) { Interlocked.Decrement(ref _backlogCurrentEnqueued); return true; @@ -1103,6 +1162,24 @@ private bool BacklogTryDequeue([NotNullWhen(true)] out Message? message) return false; } + private bool BacklogTryPeek([NotNullWhen(true)] out Message? message) + => _backlogFront.TryPeek(out message) || _backlog.TryPeek(out message); + + /// + /// Requeues, ahead of everything else, a caller's message that lost its connection to a maintenance handoff + /// while waiting for the write lock: it was issued before anything the hold has queued since, and has not + /// been written. See . + /// + private void BacklogRequeueFront(Message message) + { + Interlocked.Increment(ref _backlogCurrentEnqueued); + Interlocked.Increment(ref _backlogTotalEnqueued); + _backlogFront.Enqueue(message); + message.SetBacklogged(); + // the hold may already have been released, in which case nothing else would pick this up before the heartbeat + if (!_quiescing) KickBacklogProcessor(); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void StartBacklogProcessor() { @@ -1149,13 +1226,6 @@ private void StartBacklogProcessor() } } - /// - /// Whether a caller is waiting on something that has actually been written to the current connection, as - /// opposed to queued in the backlog. This is what a handoff drains: backlogged work is held for the - /// replacement rather than abandoned, so it must not keep the old connection alive. - /// - internal bool HasCallerWorkInFlight() => physical?.HasCallerMessagesAwaitingResponse() == true; - /// /// Whether this bridge owes a *caller* anything, ignoring our own internal traffic. /// @@ -1163,7 +1233,11 @@ internal bool HasCallerWork() { if (physical?.HasCallerMessagesAwaitingResponse() == true) return true; - foreach (var message in _backlog) // snapshot enumeration; a concurrent queue is fine to walk + foreach (var message in _backlogFront) // snapshot enumeration; a concurrent queue is fine to walk + { + if (message.IsCallerFacing) return true; + } + foreach (var message in _backlog) { if (message.IsCallerFacing) return true; } @@ -1187,7 +1261,7 @@ private void CheckBacklogForTimeouts() // Because peeking at the backlog, checking message and then dequeuing, is not thread-safe, we do have to use // a lock here, for mutual exclusion of backlog DEQUEUERS. Unfortunately. // But we reduce contention by only locking if we see something that looks timed out. - while (_backlog.TryPeek(out Message? message)) + while (BacklogTryPeek(out Message? message)) { // See if the message has pass our async timeout threshold // Note: All timed out messages must be dequeued, even when no completion is needed, to be able to dequeue and complete other timed out messages. @@ -1196,7 +1270,7 @@ private void CheckBacklogForTimeouts() { // Peek again since we didn't have lock before... // and rerun the exact same checks as above, note that it may be a different message now - if (!_backlog.TryPeek(out message)) break; + if (!BacklogTryPeek(out message)) break; if (!message.HasTimedOut(now, timeout, out var _)) break; if (!BacklogTryDequeue(out var message2) || (message != message2)) // consume it for real @@ -1251,7 +1325,7 @@ private void ProcessBacklog() { while (!isDisposed) { - if (!_backlog.IsEmpty) + if (BacklogHasItems) { // TODO: vNext handoff this backlog to another primary ("can handle everything") connection // and remove any per-server commands. This means we need to track a bit of whether something @@ -1348,7 +1422,7 @@ private void ProcessBridgeBacklog() while (IsConnected) { // check whether the backlog is empty *before* even trying to get the lock - if (_backlog.IsEmpty) return; // nothing to do + if (!BacklogHasItems) return; // nothing to do // try and get the lock; if unsuccessful, retry gotLock = _singleWriter.TryTakeSync(); @@ -1461,7 +1535,11 @@ public bool HasPendingCallerFacingItems() { if (BacklogHasItems) { - foreach (var item in _backlog) // non-consuming, thread-safe, etc + foreach (var item in _backlogFront) // non-consuming, thread-safe, etc + { + if (item.IsCallerFacing) return true; + } + foreach (var item in _backlog) { if (item.IsCallerFacing) return true; } @@ -1480,19 +1558,29 @@ private WriteResult TimedOutBeforeWrite(Message message) /// /// The connection most recently replaced by a maintenance handoff, so that a writer which was headed for it - /// can tell a planned replacement from an ordinary closure. + /// can tell a planned replacement from an ordinary closure. Written and read under the write lock. /// private PhysicalConnection? _recycledForHandoff; /// - /// Whether the connection a caller's message was headed for was replaced by a maintenance handoff while the - /// writer waited for the lock. The message has not been written, so it is queued for the replacement rather - /// than written to a connection that is gone. Only the handoff case: a write that loses its connection to an - /// ordinary closure is still reported as a closure (#3167), and a message bypassing the backlog is never - /// queued. Internal traffic is exempt as well: the handshake writes to a connection before it is current. + /// For a writer that has just taken the write lock: if the connection it was headed for was replaced by a + /// maintenance handoff while it waited, the message - not yet written - is queued for the replacement at the + /// head of the backlog, since it was issued before anything the hold has queued since. Only the handoff case: + /// a write that loses its connection to an ordinary closure is still reported as a closure (#3167), a message + /// bypassing the backlog is never queued, and internal traffic is exempt because the handshake writes to a + /// connection before it is current. /// - private bool ConnectionReplacedWhileWaiting(PhysicalConnection physical, Message message, bool bypassBacklog = false) - => !bypassBacklog && !message.IsInternalCall && !ReferenceEquals(physical, this.physical) && ReferenceEquals(physical, _recycledForHandoff); + private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, bool bypassBacklog, out WriteResult result) + { + if (bypassBacklog || message.IsInternalCall || ReferenceEquals(physical, this.physical) || !ReferenceEquals(physical, _recycledForHandoff)) + { + result = default; + return false; + } + BacklogRequeueFront(message); + result = WriteResult.Success; // queued counts as success + return true; + } /// /// This writes a message to the output stream. @@ -1536,7 +1624,7 @@ internal ValueTask WriteMessageTakingWriteLockAsync(PhysicalConnect gotLock = pending.GetAwaiter().GetResult(); // fine since we know we got a result if (!gotLock) return new ValueTask(TimedOutBeforeWrite(message)); } - if (ConnectionReplacedWhileWaiting(physical, message, bypassBacklog)) return new ValueTask(QueueOrFailMessage(message)); + if (TryRequeueForHandoff(physical, message, bypassBacklog, out var requeued)) return new ValueTask(requeued); var result = WriteMessageInsideLock(physical, message); if ((result == WriteResult.Success) & message.IsFlushRequiredAsync) { @@ -1576,7 +1664,7 @@ private async ValueTask WriteMessageTakingWriteLockAsync_Awaited( { gotLock = await pending.ForAwait(); if (!gotLock) return TimedOutBeforeWrite(message); - if (ConnectionReplacedWhileWaiting(physical, message, bypassBacklog)) return QueueOrFailMessage(message); + if (TryRequeueForHandoff(physical, message, bypassBacklog, out var requeued)) return requeued; var result = WriteMessageInsideLock(physical, message); if (result == WriteResult.Success) @@ -1968,34 +2056,6 @@ internal void SimulateConnectionFailure(SimulatedFailureType failureType) physical?.SimulateConnectionFailure(failureType); } - /// - /// Drops the current connection so that a fresh one is established. - /// - /// - /// Deliberately just a dispose: that routes through RecordConnectionFailed to - /// , which reconnects immediately by itself, so there is no new lifecycle - /// here to get wrong. Used by the MOVING handoff, where the point is to choose *when* the - /// connection is replaced - after the replacement address is known - rather than waiting for the server - /// to close it and re-resolving to whatever DNS says at that moment, which has been measured as still - /// naming the node being retired. - /// - internal bool RecycleConnection(string reason) - { - var current = physical; - if (current is null) return false; - - Multiplexer.Trace($"recycling {ConnectionType} connection: {reason}", ToString()); - - // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises - // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and - // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first - // also performs the disconnect, so the Dispose below is cleanup. - _recycledForHandoff = current; // a writer already past the hold check queues for the replacement instead - current.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); - current.Dispose(); - return true; - } - internal RedisCommand? GetActiveMessage() => Volatile.Read(ref _activeMessage)?.Command; } } diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index b65f7893a..b5ec5c401 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -471,11 +471,28 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? } /// - /// How long a handoff waits for in-flight commands before replacing the connections regardless. Ordinary - /// commands clear in a round trip; anything still in flight after this is a blocking or stuck command that will - /// not finish inside the window either, and holding everyone else's work for it would be a regression. + /// How long a handoff holds new work and waits for in-flight commands before replacing the connections + /// regardless: half the relaxed timeout (five seconds by default), bounded by what is left of the window. /// - private static readonly TimeSpan DrainCap = TimeSpan.FromSeconds(1); + /// + /// Two things pull in opposite directions. A blocking command in flight - BLPOP, XREAD BLOCK, WAIT - is failed + /// by the replacement if the wait is too short, when the server's close would have let it finish; but every + /// command the application issues during the wait is held, so a long wait is a latency spike for everyone. + /// Half the relaxed timeout lets a blocking command with a timeout up to that complete, keeps the hold well + /// inside the relaxed timeout that applies to the held work for the duration (so nothing held times out + /// because of the hold), and follows the one setting an application already uses to say how patient it is + /// prepared to be during maintenance. A cap derived from the window alone would be wrong in both directions: a + /// 15s window is not a reason to hold traffic for 15s, and a 2s window is not a reason to fail a 3s BLPOP that + /// the server would have answered. + /// + private TimeSpan DrainCap(TimeSpan budget) + { + var half = TimeSpan.FromTicks(Multiplexer.RawConfig.MaintenanceRelaxedTimeout.Ticks / 2); + return budget < half ? budget : half; + } + + /// How much longer than the cap the forced pass may wait for the write lock before giving the connection up to the server's close. + private static readonly TimeSpan LockGrace = TimeSpan.FromSeconds(1); /// /// Holds new caller work back, lets what has already been written finish, then replaces the connections. @@ -492,10 +509,12 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? /// recycle follows, and the held work is written to the replacement once it is established. /// /// - /// The wait is bounded by as well as by the window: a blocking read in flight would - /// otherwise hold every other caller's work for the whole window and still be failed at the end of it. Past - /// the cap the connections are replaced regardless, and whatever was still in flight fails as the server's - /// close would have failed it. + /// The wait is bounded by , and timeouts are relaxed for the hold so that neither the + /// held work nor a blocking command being waited for is timed out by the hold itself. Past the cap the + /// connections are replaced regardless, and whatever was still in flight fails as the server's close would + /// have failed it. Every replacement happens under the bridge's write lock, and only ever to the connection the + /// hold began on: a replacement established meanwhile is never touched, and if the lock cannot be had at all + /// the connection is left to the server's close rather than recycled around a writer. /// /// /// Both bridges, not just the one that was told. The measured blast radius is the *node*: four connections @@ -506,66 +525,50 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) { var watch = ValueStopwatch.StartNew(); - var cap = budget < DrainCap ? budget : DrainCap; + var cap = DrainCap(budget); + + // the held work, and any blocking command being waited for, must not be timed out by the hold itself + ExtendRelaxedWindow(cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout, "MOVING handoff"); + interactive?.BeginQuiesce(); subscription?.BeginQuiesce(); try { - bool drained, recycled; + using var lockDeadline = new CancellationTokenSource(cap + LockGrace); + bool? interactiveResult = interactive is null ? false : null; + bool? subscriptionResult = subscription is null ? false : null; + bool forced = false; while (true) { - var attempt = await TryRecycleWhenDrainedAsync(reason).ForAwait(); - if (attempt is { } done) - { - drained = true; - recycled = done; - break; - } - if (watch.ElapsedMilliseconds >= cap.TotalMilliseconds) + var force = watch.ElapsedMilliseconds >= cap.TotalMilliseconds; + forced |= force; + if (interactiveResult is null) interactiveResult = await interactive!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); + if (subscriptionResult is null) subscriptionResult = await subscription!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); + if (interactiveResult is not null && subscriptionResult is not null) break; + if (lockDeadline.IsCancellationRequested) { - drained = false; - recycled = (interactive?.RecycleConnection(reason) == true) | (subscription?.RecycleConnection(reason) == true); + // the write lock could not be had even for the forced pass: something is stuck on it. Recycling + // without the lock would reopen the mid-write loss the lock exists to prevent, so the connection + // is left to the server's close, which is where things stood before handoffs existed break; } await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } + var recycled = interactiveResult == true || subscriptionResult == true; if (recycled) Interlocked.Increment(ref _handoffRecycles); Multiplexer.Trace( $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" - + (drained ? " (drained)" : " (still busy at the drain cap)"), + + (forced ? " (still busy at the drain cap)" : " (drained)") + + (interactiveResult is null || subscriptionResult is null ? " (write lock not obtained; left to the server)" : ""), ToString()); } finally { - // the recycle has already nulled the old connection, so releasing the hold cannot write to it; the - // held work waits in the backlog for the replacement, as it would across any reconnect - interactive?.EndQuiesce(); - subscription?.EndQuiesce(); - } - } - - /// - /// Each bridge in turn, under its write lock: null as soon as one still has caller work in flight, - /// otherwise whether anything was recycled. A bridge recycled on an earlier pass has nothing left to recycle, - /// so a partial pass is safe to repeat. - /// - private async ValueTask TryRecycleWhenDrainedAsync(string reason) - { - bool recycled = false; - if (interactive is { } i) - { - var result = await i.TryRecycleWhenDrainedAsync(reason).ForAwait(); - if (result is null) return null; - recycled |= result.Value; - } - if (subscription is { } s) - { - var result = await s.TryRecycleWhenDrainedAsync(reason).ForAwait(); - if (result is null) return null; - recycled |= result.Value; + // each bridge is released on its own: a failure releasing one must not leave the other holding work + try { interactive?.EndQuiesce(); } catch (Exception ex) { Multiplexer.OnInternalError(ex, EndPoint); } + try { subscription?.EndQuiesce(); } catch (Exception ex) { Multiplexer.OnInternalError(ex, EndPoint); } } - return recycled; } /// diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index f7da7a062..e203139c6 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -392,13 +392,6 @@ internal bool IsIdle() internal bool HasCallerWork() => interactive?.HasCallerWork() == true || subscription?.HasCallerWork() == true; - /// - /// Whether a caller is waiting on something already written to a current connection; backlogged work does - /// not count. See . - /// - internal bool HasCallerWorkInFlight() - => interactive?.HasCallerWorkInFlight() == true || subscription?.HasCallerWorkInFlight() == true; - /// /// Work this server still owes an answer on: written-and-awaiting-response, plus anything queued in /// the backlog. Zero means a retirement can complete without abandoning anyone. diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 1c8fe4319..f503c0b90 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -913,20 +913,20 @@ await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutM } [Fact] - public async Task MovingWithSlowCommandsInFlightRecyclesAtTheDrainCap() + public async Task MovingWaitsForSlowCommandsThatFitInsideTheDrainCap() { - // A blocking or very slow command must not hold everyone else's work in the backlog for the whole window: - // the drain waits a bounded time for in-flight commands and then recycles regardless. The slow commands are - // failed by the recycle, as the server's close would have failed them; everything issued afterwards flows. + // A slow or blocking command that will finish inside the drain cap (half the relaxed timeout, 5s by default) + // is allowed to: the hold waits for it, timeouts are relaxed meanwhile, and the recycle follows its reply. + // The server's close would have let it finish too, so failing it would be a regression. var logs = new CapturingLoggerFactory(); var (server, conn, events) = await ConnectAsync(log, logs); using (server) await using (conn) { - server.SetLatency(TimeSpan.FromSeconds(3)); // every command now outlasts the drain cap + server.SetLatency(TimeSpan.FromSeconds(3)); // the in-process server answers one request at a time, so one slow command, not several var db = conn.GetDatabase(); - var slow = Enumerable.Range(0, 3).Select(_ => db.PingAsync()).ToArray(); - await Task.Delay(250); // let them be written + var slow = new[] { db.PingAsync() }; + await Task.Delay(250); // let it be written var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); @@ -936,17 +936,54 @@ public async Task MovingWithSlowCommandsInFlightRecyclesAtTheDrainCap() Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); Assert.True( - await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8_000), + await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 10_000), + "the handoff should have replaced the connection"); + var recycledAfter = watch.Elapsed; + server.SetLatency(TimeSpan.Zero); + log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with a 3s command in flight"); + + // the slow command was answered, not failed: the recycle waited for it + foreach (var pending in slow) Assert.True(await pending >= TimeSpan.Zero); + Assert.True(recycledAfter >= TimeSpan.FromSeconds(2), $"recycled after {recycledAfter}: before the in-flight command could have been answered"); + Assert.True(recycledAfter < TimeSpan.FromSeconds(6), $"recycled after {recycledAfter}: later than the cap plus jitter"); + } + } + + [Fact] + public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() + { + // A command that will not finish inside the cap - a long blocking read - must not hold everyone else's work + // in the backlog for the whole window: at the cap the connection is replaced regardless. The blocking command + // is failed by the recycle, as the server's close would have failed it; everything issued afterwards flows. + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); + using (server) + await using (conn) + { + server.SetLatency(TimeSpan.FromSeconds(9)); // longer than the 5s cap; shorter than the relaxed 10s timeout + var db = conn.GetDatabase(); + var blocking = Enumerable.Range(0, 3).Select(_ => db.PingAsync()).ToArray(); + await Task.Delay(250); // let them be written + + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + var watch = System.Diagnostics.Stopwatch.StartNew(); + server.SendMoving(null, timeSeconds: 15, newEndpoint: successor, sequenceId: 0); + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + Assert.True( + await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 12_000), "the handoff should have replaced the connection"); var recycledAfter = watch.Elapsed; server.SetLatency(TimeSpan.Zero); // so the replacement's handshake is not slowed too - log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with slow commands in flight"); + log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with 9s commands in flight"); - // jitter is at most a tenth of the window, the cap one second: well short of the 3s the commands need - Assert.True(recycledAfter < TimeSpan.FromSeconds(3), $"recycled after {recycledAfter}: the drain waited for the slow commands instead of capping"); + // the cap (5s) plus at most a second of jitter: well short of the 9s the commands need + Assert.True(recycledAfter < TimeSpan.FromSeconds(7), $"recycled after {recycledAfter}: the drain waited for the blocking commands instead of capping"); // the in-flight commands were failed by the recycle, which is what the server's close would have done - foreach (var pending in slow) + foreach (var pending in blocking) { await Assert.ThrowsAnyAsync(async () => await pending); } @@ -964,7 +1001,7 @@ await Poll.UntilAsync( return false; } }, - timeoutMilliseconds: 30_000), + timeoutMilliseconds: 40_000), "the client should be serving commands on the replacement connection"); } } From bd786f3bbec5d21901daf419b4a824ab5ea054ba Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 4 Nov 2026 16:38:04 +0000 Subject: [PATCH 04/27] Handoff drain: head inserts under the backlog lock, marker cleared only at the next hold, per-bridge release, cap floor Third round of review follow-ups: - BacklogRequeueFront takes lock(_backlog) around the head insert. The timeout sweep peeks, then dequeues and compares under that lock; an unlocked insert between the two handed it a different message, which it treats as a threading bug, and the dequeued message was dropped uncompleted. - _recycledForHandoff is cleared when the next hold begins, not when the replacement establishes: the write lock is not fair, so a writer parked on it can be bypassed by the backlog processor long enough for the replacement to come up, and clearing then made it write to the disposed connection and tear down the healthy replacement. - DrainCap has a one-second floor, so a relaxed timeout of zero (relaxation disabled) does not force the recycle with commands still in flight. - Each bridge is released as its own retirement resolves, not when both have; the interactive bridge usually drains in a round trip and must not hold work while the subscription bridge takes longer. EndQuiesce is idempotent and kicks the processor unconditionally on the transition, closing the store-load race with a concurrent enqueuer. - The drain works on snapshots of the bridge fields, and the mutex take and release tolerate disposal mid-handoff. - A batch carrying a QUIT is not held at all, so the hold decision and the per-message action agree. - The lockDeadline comment no longer claims the mutex timeout plays no part. New test: the steady-load handoff with maintRelaxedTimeout=0. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 46 +++++++++++++++---- .../ServerEndPoint.Maintenance.cs | 40 ++++++++++++---- .../MaintenanceNotificationTests.cs | 16 +++++-- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 7fcb8201b..4579d9e20 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default); timeouts are relaxed for its duration, so nothing is timed out by the hold itself, and a blocking command with a timeout up to that completes normally. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default, and never less than a second); timeouts are relaxed for its duration, so nothing is timed out by the hold itself, and a blocking command with a timeout up to that completes normally. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 9603b3958..2f4d1b773 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -158,6 +158,7 @@ public enum State : byte /// internal void BeginQuiesce() { + _recycledForHandoff = null; // the previous handoff's marker; any writer parked across two handoffs is on its own _quiesceTarget = physical; _quiesceRecycled = false; _quiescing = true; @@ -165,9 +166,13 @@ internal void BeginQuiesce() internal void EndQuiesce() { + if (!_quiescing) return; _quiescing = false; _quiesceTarget = null; - if (BacklogHasItems) KickBacklogProcessor(); + // Unconditionally, not only when the backlog has items: an enqueuer that read the flag just before it + // flipped did not kick, and neither side's volatile access orders its store before the other's load. A + // kick with nothing queued costs one idle pass of the processor. + KickBacklogProcessor(); } /// Gets held or queued work moving: the processor may be lingering between passes, or may have exited. @@ -203,7 +208,8 @@ private bool ShouldHoldForHandoff(Message message) /// /// Replace the connection even if caller work is still in flight on it. /// Why; for the trace. - /// Bounds the wait for the write lock: the drain decides how long, not the mutex timeout. + /// Bounds the wait for the write lock (together with the mutex's own timeout, whichever + /// ends first); the drain re-waits until this deadline, so it is the deadline that decides how long overall. /// null if the connection could not be retired yet - caller work in flight, or the lock not /// obtained inside the deadline - so the drain should try again; otherwise whether this call, or an earlier /// one, replaced it. @@ -222,6 +228,10 @@ private bool ShouldHoldForHandoff(Message message) { gotLock = false; } + catch (ObjectDisposedException) + { + return false; // the bridge was disposed mid-handoff; there is nothing left to retire + } if (!gotLock) return null; try { @@ -249,7 +259,14 @@ private bool ShouldHoldForHandoff(Message message) } finally { - _singleWriter.Release(); + try + { + _singleWriter.Release(); + } + catch (ObjectDisposedException) + { + // disposed while we held it; nobody is waiting on a disposed mutex + } } } @@ -690,7 +707,6 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) { reportNextFailure = reconfigureNextFailure = true; LastException = null; - _recycledForHandoff = null; Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -904,10 +920,11 @@ internal bool TryEnqueue(List messages, bool isReplica) if (isDisposed) throw new ObjectDisposedException(Name); - if (_quiescing && IsConnected && !NeedsReconnect && messages.Exists(m => !m.IsInternalCall && m.Command != RedisCommand.QUIT)) + if (_quiescing && IsConnected && !NeedsReconnect && messages.Exists(m => !m.IsInternalCall) && !messages.Exists(m => m.Command == RedisCommand.QUIT)) { // a maintenance handoff is draining the connection: hold the batch for the replacement. A batch is - // written as a unit, so it is held as a unit; a batch of purely internal traffic is not held at all + // written as a unit, so it is held as a unit; a batch of purely internal traffic is not held at all, + // and nor is one carrying a QUIT, which is never queued - the whole batch goes to the live connection foreach (var message in messages) { QueueOrFailMessage(message, holdForHandoff: true); @@ -1174,10 +1191,16 @@ private void BacklogRequeueFront(Message message) { Interlocked.Increment(ref _backlogCurrentEnqueued); Interlocked.Increment(ref _backlogTotalEnqueued); - _backlogFront.Enqueue(message); + // Under the dequeuers' lock: CheckBacklogForTimeouts peeks, then dequeues and compares under this lock, + // and an insert at the head between the two would hand it a different message - which it treats as a + // threading bug, and the dequeued message would be dropped uncompleted. Only locked parties change the head. + lock (_backlog) + { + _backlogFront.Enqueue(message); + } message.SetBacklogged(); - // the hold may already have been released, in which case nothing else would pick this up before the heartbeat - if (!_quiescing) KickBacklogProcessor(); + // unconditionally: the hold may be released concurrently, and a kick with the hold still on costs one idle pass + KickBacklogProcessor(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -1558,7 +1581,10 @@ private WriteResult TimedOutBeforeWrite(Message message) /// /// The connection most recently replaced by a maintenance handoff, so that a writer which was headed for it - /// can tell a planned replacement from an ordinary closure. Written and read under the write lock. + /// can tell a planned replacement from an ordinary closure. Set under the write lock when the connection is + /// retired, read under it by the writer, and cleared only when the next hold begins: the writer may have + /// been waiting for the lock long enough for the replacement to be fully established, since the lock is not + /// fair and the backlog processor may keep taking it to flush held work. /// private PhysicalConnection? _recycledForHandoff; diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index b5ec5c401..6894ebc82 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -488,9 +488,13 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? private TimeSpan DrainCap(TimeSpan budget) { var half = TimeSpan.FromTicks(Multiplexer.RawConfig.MaintenanceRelaxedTimeout.Ticks / 2); + if (half < DrainFloor) half = DrainFloor; // a relaxed timeout of zero disables relaxation; it must not disable the drain return budget < half ? budget : half; } + /// The least the drain waits for in-flight commands, whatever the relaxed timeout: one round trip on any network, with room to spare. + private static readonly TimeSpan DrainFloor = TimeSpan.FromSeconds(1); + /// How much longer than the cap the forced pass may wait for the write lock before giving the connection up to the server's close. private static readonly TimeSpan LockGrace = TimeSpan.FromSeconds(1); @@ -530,20 +534,25 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) // the held work, and any blocking command being waited for, must not be timed out by the hold itself ExtendRelaxedWindow(cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout, "MOVING handoff"); - interactive?.BeginQuiesce(); - subscription?.BeginQuiesce(); + // snapshots: Dispose nulls the fields, and a handoff in progress must not fault on a server going away + var interactiveBridge = interactive; + var subscriptionBridge = subscription; + interactiveBridge?.BeginQuiesce(); + subscriptionBridge?.BeginQuiesce(); try { using var lockDeadline = new CancellationTokenSource(cap + LockGrace); - bool? interactiveResult = interactive is null ? false : null; - bool? subscriptionResult = subscription is null ? false : null; + bool? interactiveResult = interactiveBridge is null ? false : null; + bool? subscriptionResult = subscriptionBridge is null ? false : null; bool forced = false; while (true) { var force = watch.ElapsedMilliseconds >= cap.TotalMilliseconds; forced |= force; - if (interactiveResult is null) interactiveResult = await interactive!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); - if (subscriptionResult is null) subscriptionResult = await subscription!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); + // each bridge is released the moment its own connection is settled: the interactive bridge usually + // drains in a round trip, and must not keep holding work while the subscription bridge takes longer + if (interactiveResult is null && (interactiveResult = await interactiveBridge!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait()) is not null) Release(interactiveBridge); + if (subscriptionResult is null && (subscriptionResult = await subscriptionBridge!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait()) is not null) Release(subscriptionBridge); if (interactiveResult is not null && subscriptionResult is not null) break; if (lockDeadline.IsCancellationRequested) { @@ -565,9 +574,22 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) } finally { - // each bridge is released on its own: a failure releasing one must not leave the other holding work - try { interactive?.EndQuiesce(); } catch (Exception ex) { Multiplexer.OnInternalError(ex, EndPoint); } - try { subscription?.EndQuiesce(); } catch (Exception ex) { Multiplexer.OnInternalError(ex, EndPoint); } + // whatever was not released in the loop (the lock-not-obtained exit, an exception): EndQuiesce is idempotent + Release(interactiveBridge); + Release(subscriptionBridge); + } + + void Release(PhysicalBridge? bridge) + { + if (bridge is null) return; + try + { + bridge.EndQuiesce(); + } + catch (Exception ex) + { + Multiplexer.OnInternalError(ex, EndPoint); // one bridge failing to release must not leave the other holding work + } } } diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index f503c0b90..aa1de859e 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -27,13 +27,15 @@ public class MaintenanceNotificationTests(ITestOutputHelper log) /// private static async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection, EventCollector Events)> ConnectAsync( ITestOutputHelper log, - ILoggerFactory? loggerFactory = null) + ILoggerFactory? loggerFactory = null, + Action? configure = null) { var server = new InProcessTestServer(log); var config = server.GetClientConfig(defaultOnly: true); config.Protocol = RedisProtocol.Resp3; config.MaintenanceNotifications = MaintenanceNotificationMode.Enabled; // must be live, or the test is vacuous config.LoggerFactory = loggerFactory; + configure?.Invoke(config); var conn = await ConnectionMultiplexer.ConnectAsync(config); return (server, conn, new EventCollector(conn)); @@ -844,7 +846,15 @@ await Poll.UntilAsync( } [Fact] - public async Task MovingUnderSteadyLoadRecyclesPromptlyAndFailsNothing() + public Task MovingUnderSteadyLoadRecyclesPromptlyAndFailsNothing() => SteadyLoadHandoffAsync(configure: null); + + [Fact] + public Task MovingUnderSteadyLoadWithRelaxationDisabledStillDrains() + // a relaxed timeout of zero is the documented way to turn relaxation off; the drain cap is derived from it, + // and must keep a floor rather than recycle at once with commands still in flight + => SteadyLoadHandoffAsync(configure: c => c.MaintenanceRelaxedTimeout = TimeSpan.Zero); + + private async Task SteadyLoadHandoffAsync(Action? configure) { // The drain used to wait for "no caller work", which under steady traffic never comes: measured live at // about 450 ops/s, the recycle arrived at the end of the 15s window and failed the eight commands in @@ -852,7 +862,7 @@ public async Task MovingUnderSteadyLoadRecyclesPromptlyAndFailsNothing() // written commands finish, so the recycle follows within a round trip and nothing fails: the held work is // written to the replacement. var logs = new CapturingLoggerFactory(); - var (server, conn, events) = await ConnectAsync(log, logs); + var (server, conn, events) = await ConnectAsync(log, logs, configure); using (server) await using (conn) { From a518499eca06070bf8a03991a2d783cf739e059a Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Wed, 4 Nov 2026 17:16:08 +0000 Subject: [PATCH 05/27] Handoff drain: survive the old connection dying mid-hold, cover held work's timeout, budget from what is left Fourth round of review follow-ups: - A connection failure during the hold no longer abandons the held work under a FailFast backlog policy: that backlog is work diverted off a live connection, which would have been written and answered, so it is left for the replacement to flush. Previously only the handoff's own recycle was exempt. - The hold imposes its own floor on the effective timeout for its duration plus the reconnect (the cap, the lock grace and the connect timeout), independent of relaxation: with maintRelaxedTimeout=0 and a sub-second configured timeout, the one-second drain floor would otherwise have timed out exactly the work the hold protects. - The front requeue roots the multiplexer on the empty-to-non-empty transition, as the ordinary enqueue does. - The drain budget is what is left of the window after the decision has spent its share polling DNS, not the window minus jitter; with less than the floor left there is no hold, and the connections are replaced at once (still under the lock) so the replacement gets what remains. - The handoff marker is a short ring of the last four retired connections, never cleared, so a writer starved across back-to-back MOVINGs still recognises its connection. - Abandoning the backlog dequeues under the dequeuers' lock; a quiesced backlog pass returns before contending for the write lock; the processor start's wake tolerates disposal. - One hold predicate, shared by the single-message gates and the batch gate, which makes one pass over the batch. - The docs and the DrainCap remarks now agree with the clamp: the window bounds the hold because the server closes the socket when it ends. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 86 +++++++++++++++---- .../ServerEndPoint.Maintenance.cs | 66 +++++++++++--- 3 files changed, 124 insertions(+), 30 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 4579d9e20..13d05eb8b 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default, and never less than a second); timeouts are relaxed for its duration, so nothing is timed out by the hold itself, and a blocking command with a timeout up to that completes normally. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default, never less than a second, and never longer than what is left of the announced window, since the server closes the socket when that ends regardless). For the duration of the hold and the reconnect after it, commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 2f4d1b773..a44eef52c 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -158,7 +158,6 @@ public enum State : byte /// internal void BeginQuiesce() { - _recycledForHandoff = null; // the previous handoff's marker; any writer parked across two handoffs is on its own _quiesceTarget = physical; _quiesceRecycled = false; _quiescing = true; @@ -195,8 +194,28 @@ private void KickBacklogProcessor() /// rules and the backlog policy apply), and only for a caller's work. Internal traffic (keep-alives, probes) /// still goes to the old connection, and so does QUIT, which is never queued. /// - private bool ShouldHoldForHandoff(Message message) - => _quiescing && IsConnected && !NeedsReconnect && !message.IsInternalCall && message.Command != RedisCommand.QUIT; + private bool ShouldHoldForHandoff(Message message) => HoldActive && IsHoldable(message); + + /// A handoff is draining a connection that is still live. + private bool HoldActive => _quiescing && IsConnected && !NeedsReconnect; + + /// A caller's work, and not QUIT, which is never queued. + private static bool IsHoldable(Message message) => !message.IsInternalCall && message.Command != RedisCommand.QUIT; + + /// + /// A batch is written as a unit, so it is held as a unit: when it carries any holdable message and no QUIT. + /// An internal member rides along with the rest, because splitting the batch would be worse than holding it. + /// + private static bool ShouldHoldBatch(List messages) + { + bool any = false; + foreach (var message in messages) + { + if (message.Command == RedisCommand.QUIT) return false; + any |= IsHoldable(message); + } + return any; + } /// /// Retires the connection the hold began on, under the write lock, once nothing a caller is waiting on is in @@ -251,7 +270,7 @@ private bool ShouldHoldForHandoff(Message message) // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first // also performs the disconnect, so the Dispose below is cleanup. - _recycledForHandoff = target; // a writer already past the hold check queues for the replacement instead + _recycledForHandoff[_recycledForHandoffNext++ & 3] = target; // a writer already past the hold check queues for the replacement instead _quiesceRecycled = true; target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); target.Dispose(); @@ -621,8 +640,10 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur } Trace($"OnConnectionFailed: {connection}"); // If we're configured to, fail all pending backlogged messages - but not for a maintenance handoff, which - // is a replacement we chose, and whose backlog is work deliberately held for the replacement connection - if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff) + // is a replacement we chose, and not while a handoff is holding work back: that backlog is work diverted + // off a live connection, which would have been written and answered, so a death of the old connection + // during the hold (the window expiring, a blip) must leave it for the replacement to flush + if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !_quiescing) { AbandonPendingBacklog(innerException, connection); } @@ -686,8 +707,13 @@ internal void OnDisconnected(ConnectionFailureType failureType, PhysicalConnecti private void AbandonPendingBacklog(Exception ex, PhysicalConnection? connection) { - while (BacklogTryDequeue(out Message? next)) + while (true) { + Message? next; + lock (_backlog) // only locked parties change the head: the timeout sweep peeks, then dequeues and compares under this lock + { + if (!BacklogTryDequeue(out next)) break; + } // as in PhysicalConnection.RecordMessageFailed: don't hand the same exception to every // message. A backlogged message has provably never been written, which is exactly what the // retry machinery needs to know, and it is lost if we share the connection-level instance. @@ -920,11 +946,9 @@ internal bool TryEnqueue(List messages, bool isReplica) if (isDisposed) throw new ObjectDisposedException(Name); - if (_quiescing && IsConnected && !NeedsReconnect && messages.Exists(m => !m.IsInternalCall) && !messages.Exists(m => m.Command == RedisCommand.QUIT)) + if (HoldActive && ShouldHoldBatch(messages)) { - // a maintenance handoff is draining the connection: hold the batch for the replacement. A batch is - // written as a unit, so it is held as a unit; a batch of purely internal traffic is not held at all, - // and nor is one carrying a QUIT, which is never queued - the whole batch goes to the live connection + // a maintenance handoff is draining the connection: hold the batch for the replacement foreach (var message in messages) { QueueOrFailMessage(message, holdForHandoff: true); @@ -1189,6 +1213,7 @@ private bool BacklogTryPeek([NotNullWhen(true)] out Message? message) /// private void BacklogRequeueFront(Message message) { + bool wasEmpty = !BacklogHasItems; Interlocked.Increment(ref _backlogCurrentEnqueued); Interlocked.Increment(ref _backlogTotalEnqueued); // Under the dequeuers' lock: CheckBacklogForTimeouts peeks, then dequeues and compares under this lock, @@ -1199,6 +1224,7 @@ private void BacklogRequeueFront(Message message) _backlogFront.Enqueue(message); } message.SetBacklogged(); + if (wasEmpty) Multiplexer.Root(); // as BacklogEnqueue: a backlog with a caller's task in it must pin the multiplexer // unconditionally: the hold may be released concurrently, and a kick with the hold still on costs one idle pass KickBacklogProcessor(); } @@ -1245,7 +1271,14 @@ private void StartBacklogProcessor() } else { - _backlogAutoReset.Set(); + try + { + _backlogAutoReset.Set(); + } + catch (ObjectDisposedException) + { + // disposed between the check and the wake; the processor that was running is winding down anyway + } } } @@ -1447,6 +1480,10 @@ private void ProcessBridgeBacklog() // check whether the backlog is empty *before* even trying to get the lock if (!BacklogHasItems) return; // nothing to do + // a maintenance handoff is holding work back: nothing would be written, and the lock is better + // left to the drain and the writers it is waiting on; the release kicks this again + if (_quiescing) return; + // try and get the lock; if unsuccessful, retry gotLock = _singleWriter.TryTakeSync(); if (gotLock) break; // got the lock; now go do something with it @@ -1580,13 +1617,24 @@ private WriteResult TimedOutBeforeWrite(Message message) } /// - /// The connection most recently replaced by a maintenance handoff, so that a writer which was headed for it - /// can tell a planned replacement from an ordinary closure. Set under the write lock when the connection is - /// retired, read under it by the writer, and cleared only when the next hold begins: the writer may have - /// been waiting for the lock long enough for the replacement to be fully established, since the lock is not - /// fair and the backlog processor may keep taking it to flush held work. + /// The last few connections replaced by maintenance handoffs, so that a writer which was headed for one of + /// them can tell a planned replacement from an ordinary closure. Written and read under the write lock, and + /// never cleared: the lock is not fair, so a writer parked on it can be starved past the replacement being + /// established, and rolling maintenance is a series of MOVINGs, so past the *next* handoff as well. Four is + /// more handoffs than a writer can realistically be starved across; older entries are simply overwritten. /// - private PhysicalConnection? _recycledForHandoff; + private readonly PhysicalConnection?[] _recycledForHandoff = new PhysicalConnection?[4]; + private int _recycledForHandoffNext; + + /// Whether a maintenance handoff retired this connection; under the write lock. + private bool WasRecycledForHandoff(PhysicalConnection connection) + { + foreach (var recycled in _recycledForHandoff) + { + if (ReferenceEquals(connection, recycled)) return true; + } + return false; + } /// /// For a writer that has just taken the write lock: if the connection it was headed for was replaced by a @@ -1598,7 +1646,7 @@ private WriteResult TimedOutBeforeWrite(Message message) /// private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, bool bypassBacklog, out WriteResult result) { - if (bypassBacklog || message.IsInternalCall || ReferenceEquals(physical, this.physical) || !ReferenceEquals(physical, _recycledForHandoff)) + if (bypassBacklog || message.IsInternalCall || ReferenceEquals(physical, this.physical) || !WasRecycledForHandoff(physical)) { result = default; return false; diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 6894ebc82..953903688 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -198,10 +198,38 @@ internal MaintenanceNotificationType ActiveMaintenanceType /// internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) { - if (GetRelaxedRemaining() <= 0) return configuredMilliseconds; + var result = configuredMilliseconds; + if (GetRelaxedRemaining() > 0) + { + var relaxed = (int)Multiplexer.RawConfig.MaintenanceRelaxedTimeout.TotalMilliseconds; + if (relaxed > result) result = relaxed; + } - var relaxed = (int)Multiplexer.RawConfig.MaintenanceRelaxedTimeout.TotalMilliseconds; - return relaxed > configuredMilliseconds ? relaxed : configuredMilliseconds; + // a handoff hold diverts commands off a live connection into the backlog; whatever the relaxation setting + // (zero disables it), those commands must not be timed out by the hold itself, so the hold imposes its + // own floor on the effective timeout for as long as it, and the reconnect after it, can take + var cover = Volatile.Read(ref _holdCoverDeadlineTicks); + if (cover != 0 && unchecked(cover - Environment.TickCount) > 0) + { + var floor = Volatile.Read(ref _holdCoverMilliseconds); + if (floor > result) result = floor; + } + return result; + } + + private int _holdCoverDeadlineTicks, _holdCoverMilliseconds; + + /// + /// For the duration of a handoff hold plus the reconnect that follows it, commands against this server time out + /// no sooner than that whole span. Independent of relaxation: a relaxed timeout of zero is the documented way to + /// turn relaxation off, and a sub-second configured timeout with the hold's one-second floor would otherwise + /// time out exactly the work the hold is protecting. + /// + private void SetHoldCover(TimeSpan span) + { + var ms = (int)span.TotalMilliseconds; + Volatile.Write(ref _holdCoverMilliseconds, ms); + Volatile.Write(ref _holdCoverDeadlineTicks, NudgeFromZero(unchecked(Environment.TickCount + ms))); } /// @@ -422,6 +450,10 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? resolve: Multiplexer.AddressResolver, log: message => Multiplexer.Trace(message, ToString())).ForAwait(); + // the decision may have spent a good part of the window polling DNS (4 to 19 seconds of a 15 second + // window has been measured); what the drain has to work with is what is actually left + remaining = Remaining(window, watch); + Multiplexer.Trace($"MOVING: {decision}", ToString()); _lastHandoffOutcome = decision.ToString(); @@ -440,7 +472,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // go on. Half of the *announced* window, less whatever the jitter already spent. var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); - await DrainThenRecycleAsync(window - jitter - (half > TimeSpan.Zero ? half : TimeSpan.Zero), decision.Reason).ForAwait(); + await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason).ForAwait(); replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: @@ -470,6 +502,13 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? } } + /// What is left of an announced window; never negative. + private static TimeSpan Remaining(TimeSpan window, ValueStopwatch watch) + { + var left = window - TimeSpan.FromMilliseconds(watch.ElapsedMilliseconds); + return left > TimeSpan.Zero ? left : TimeSpan.Zero; + } + /// /// How long a handoff holds new work and waits for in-flight commands before replacing the connections /// regardless: half the relaxed timeout (five seconds by default), bounded by what is left of the window. @@ -481,9 +520,10 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? /// Half the relaxed timeout lets a blocking command with a timeout up to that complete, keeps the hold well /// inside the relaxed timeout that applies to the held work for the duration (so nothing held times out /// because of the hold), and follows the one setting an application already uses to say how patient it is - /// prepared to be during maintenance. A cap derived from the window alone would be wrong in both directions: a - /// 15s window is not a reason to hold traffic for 15s, and a 2s window is not a reason to fail a 3s BLPOP that - /// the server would have answered. + /// prepared to be during maintenance. The window still bounds it, because the server closes the socket when + /// the window ends whatever we do: a command that cannot finish inside the window fails either way, and holding + /// past it would only delay everything else. A cap derived from the window alone would be wrong the other way: + /// a 15s window is not a reason to hold traffic for 15s. /// private TimeSpan DrainCap(TimeSpan budget) { @@ -529,10 +569,16 @@ private TimeSpan DrainCap(TimeSpan budget) private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) { var watch = ValueStopwatch.StartNew(); - var cap = DrainCap(budget); - - // the held work, and any blocking command being waited for, must not be timed out by the hold itself + // With less than the floor left of the window there is no hold worth having: the server is about to close + // the socket regardless, so the connections are replaced at once (still under the lock) and the replacement + // gets what little time remains. Otherwise the hold lasts up to the cap. + var cap = budget < DrainFloor ? TimeSpan.Zero : DrainCap(budget); + + // the held work, and any blocking command being waited for, must not be timed out by the hold itself, nor + // by the reconnect that follows it; the relaxed window covers it where relaxation is on, and the cover below + // covers it regardless ExtendRelaxedWindow(cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout, "MOVING handoff"); + SetHoldCover(cap + LockGrace + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout)); // snapshots: Dispose nulls the fields, and a handoff in progress must not fault on a server going away var interactiveBridge = interactive; From 04c67614e044bdad151ae285807e2eb4029ce0b7 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 09:21:09 +0000 Subject: [PATCH 06/27] Handoff drain: cover the pinned successor, cap at half the remaining window, exact retired marker Fifth round of review follow-ups: - The hold cover spans the rest of the window plus one connect attempt, not one connect attempt alone: a MoveTo pins reconnection to the named successor until the window ends, and a successor that does not answer is tried until then. With relaxation disabled, held work could outlive the cover and be swept. - The drain cap is also bounded by half of what is left of the window, so a relaxed timeout of two windows or more cannot turn the hold into a full-window stall that recycles at the deadline with work in flight. - A disposed wake in KickBacklogProcessor still starts the processor, whose disposed-drain is the only thing that fails what is left once the heartbeat has stopped. - The hold is keyed on the quiesce target still being the current connection, not on IsConnected: once the target is gone, by the drain or by itself, new work and the backlog flow to whatever is current. - The retired-for-handoff marker is a flag on the connection itself, exact and holding nothing, in place of the four-slot ring. - The finalizer abandons the backlog without the dequeuers' lock; everything else keeps taking it. - The steady-load tests' resumption poll is wider, with a note that the in-process tunnel never dials the successor. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 74 +++++++++---------- src/StackExchange.Redis/PhysicalConnection.cs | 9 +++ .../ServerEndPoint.Maintenance.cs | 30 +++++--- .../MaintenanceNotificationTests.cs | 8 +- 5 files changed, 72 insertions(+), 51 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 13d05eb8b..5e642a25b 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default, never less than a second, and never longer than what is left of the announced window, since the server closes the socket when that ends regardless). For the duration of the hold and the reconnect after it, commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second, and never longer than the window itself, since the server closes the socket when that ends regardless. For the duration of the hold and the reconnect after it, commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index a44eef52c..f428f6bbc 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -183,7 +183,8 @@ private void KickBacklogProcessor() } catch (ObjectDisposedException) { - return; // a bridge disposed mid-handoff has no processor to wake, and nothing left to write + // disposed: nothing to wake, but fall through - a processor started now runs the disposed-drain that + // fails whatever is queued, which is the only thing that will, since the heartbeat has stopped } StartBacklogProcessor(); } @@ -196,8 +197,15 @@ private void KickBacklogProcessor() /// private bool ShouldHoldForHandoff(Message message) => HoldActive && IsHoldable(message); - /// A handoff is draining a connection that is still live. - private bool HoldActive => _quiescing && IsConnected && !NeedsReconnect; + /// + /// A handoff is draining the connection that is current right now. The moment the target is gone - retired + /// by the drain, or dead of its own accord with a replacement already establishing - there is nothing to hold + /// work back from, and the backlog may flush to whatever is current. + /// + private bool HoldingCurrentConnection => _quiescing && _quiesceTarget is { } target && ReferenceEquals(physical, target); + + /// A handoff is draining the current connection, and it is live. + private bool HoldActive => HoldingCurrentConnection && IsConnected && !NeedsReconnect; /// A caller's work, and not QUIT, which is never queued. private static bool IsHoldable(Message message) => !message.IsInternalCall && message.Command != RedisCommand.QUIT; @@ -270,7 +278,7 @@ private static bool ShouldHoldBatch(List messages) // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first // also performs the disconnect, so the Dispose below is cleanup. - _recycledForHandoff[_recycledForHandoffNext++ & 3] = target; // a writer already past the hold check queues for the replacement instead + target.MarkRetiredForHandoff(); // a writer already past the hold check queues for the replacement instead (see TryRequeueForHandoff) _quiesceRecycled = true; target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); target.Dispose(); @@ -312,7 +320,7 @@ public void Dispose() isDisposed = true; // make damn sure we don't true to resurrect // If there's anything in the backlog and we're being torn down - exfil it immediately (e.g. so all awaitables complete) - AbandonPendingBacklog(new ObjectDisposedException("Connection is being finalized"), null); + AbandonPendingBacklog(new ObjectDisposedException("Connection is being finalized"), null, locked: false); // shouldn't *really* touch managed objects // in a finalizer, but we need to kill that socket, @@ -705,14 +713,25 @@ internal void OnDisconnected(ConnectionFailureType failureType, PhysicalConnecti } } - private void AbandonPendingBacklog(Exception ex, PhysicalConnection? connection) + /// Why. + /// The connection, if any, to report against. + /// Take the dequeuers' lock, as every head-changing party must while anything else can run; + /// the finalizer passes false, since it must not block and nothing else is running by then. + private void AbandonPendingBacklog(Exception ex, PhysicalConnection? connection, bool locked = true) { while (true) { Message? next; - lock (_backlog) // only locked parties change the head: the timeout sweep peeks, then dequeues and compares under this lock + if (locked) { - if (!BacklogTryDequeue(out next)) break; + lock (_backlog) // only locked parties change the head: the timeout sweep peeks, then dequeues and compares under this lock + { + if (!BacklogTryDequeue(out next)) break; + } + } + else if (!BacklogTryDequeue(out next)) + { + break; } // as in PhysicalConnection.RecordMessageFailed: don't hand the same exception to every // message. A backlogged message has provably never been written, which is exactly what the @@ -1480,9 +1499,9 @@ private void ProcessBridgeBacklog() // check whether the backlog is empty *before* even trying to get the lock if (!BacklogHasItems) return; // nothing to do - // a maintenance handoff is holding work back: nothing would be written, and the lock is better - // left to the drain and the writers it is waiting on; the release kicks this again - if (_quiescing) return; + // a maintenance handoff is holding work back from this connection: nothing would be written, and + // the lock is better left to the drain and the writers it is waiting on; the release kicks this again + if (HoldingCurrentConnection) return; // try and get the lock; if unsuccessful, retry gotLock = _singleWriter.TryTakeSync(); @@ -1494,7 +1513,7 @@ private void ProcessBridgeBacklog() // Timeouts are handled above, so we're exclusively into backlog items eligible to write at this point. // If we can't write them, abort and wait for the next heartbeat or activation to try this again. bool flush = false; - while (IsConnected && !_quiescing) // not while a handoff is holding work back for the replacement + while (IsConnected && !HoldingCurrentConnection) // not while a handoff is holding work back from this connection { // Snapshot the connection for the whole of this message: OnDisconnected nulls the field // without waiting for the write lock, so re-reading it (as the loop guard used to) can hand @@ -1616,37 +1635,18 @@ private WriteResult TimedOutBeforeWrite(Message message) return WriteResult.TimeoutBeforeWrite; } - /// - /// The last few connections replaced by maintenance handoffs, so that a writer which was headed for one of - /// them can tell a planned replacement from an ordinary closure. Written and read under the write lock, and - /// never cleared: the lock is not fair, so a writer parked on it can be starved past the replacement being - /// established, and rolling maintenance is a series of MOVINGs, so past the *next* handoff as well. Four is - /// more handoffs than a writer can realistically be starved across; older entries are simply overwritten. - /// - private readonly PhysicalConnection?[] _recycledForHandoff = new PhysicalConnection?[4]; - private int _recycledForHandoffNext; - - /// Whether a maintenance handoff retired this connection; under the write lock. - private bool WasRecycledForHandoff(PhysicalConnection connection) - { - foreach (var recycled in _recycledForHandoff) - { - if (ReferenceEquals(connection, recycled)) return true; - } - return false; - } - /// /// For a writer that has just taken the write lock: if the connection it was headed for was replaced by a /// maintenance handoff while it waited, the message - not yet written - is queued for the replacement at the - /// head of the backlog, since it was issued before anything the hold has queued since. Only the handoff case: - /// a write that loses its connection to an ordinary closure is still reported as a closure (#3167), a message - /// bypassing the backlog is never queued, and internal traffic is exempt because the handshake writes to a - /// connection before it is current. + /// head of the backlog, since it was issued before anything the hold has queued since. The connection itself + /// says whether a handoff retired it (), so a writer starved + /// across any number of later handoffs still recognises its own. Only the handoff case: a write that loses its + /// connection to an ordinary closure is still reported as a closure (#3167), a message bypassing the backlog is + /// never queued, and internal traffic is exempt because the handshake writes to a connection before it is current. /// private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, bool bypassBacklog, out WriteResult result) { - if (bypassBacklog || message.IsInternalCall || ReferenceEquals(physical, this.physical) || !WasRecycledForHandoff(physical)) + if (bypassBacklog || message.IsInternalCall || ReferenceEquals(physical, this.physical) || !physical.RetiredForHandoff) { result = default; return false; diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 9e5eab6a1..41047ba03 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -990,6 +990,15 @@ internal void GetStormLog(StringBuilder sb) /// of walking a queue that a stalled server can leave thousands of entries long. /// /// + /// + /// Whether a maintenance handoff retired this connection, so that a writer which was headed for it and only + /// reached the write lock afterwards can tell a planned replacement from an ordinary closure. Set under the + /// bridge's write lock, immediately before the connection is disposed; never reset. + /// + internal bool RetiredForHandoff { get; private set; } + + internal void MarkRetiredForHandoff() => RetiredForHandoff = true; + internal bool HasCallerMessagesAwaitingResponse() { lock (_writtenAwaitingResponse) diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 953903688..a382eecff 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -220,8 +220,9 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) private int _holdCoverDeadlineTicks, _holdCoverMilliseconds; /// - /// For the duration of a handoff hold plus the reconnect that follows it, commands against this server time out - /// no sooner than that whole span. Independent of relaxation: a relaxed timeout of zero is the documented way to + /// For the duration of a handoff hold plus the reconnect that follows it - including the rest of the window, for + /// which a MoveTo pins reconnection to the named successor - commands against this server time out no sooner + /// than that whole span. Independent of relaxation: a relaxed timeout of zero is the documented way to /// turn relaxation off, and a sub-second configured timeout with the hold's one-second floor would otherwise /// time out exactly the work the hold is protecting. /// @@ -511,7 +512,8 @@ private static TimeSpan Remaining(TimeSpan window, ValueStopwatch watch) /// /// How long a handoff holds new work and waits for in-flight commands before replacing the connections - /// regardless: half the relaxed timeout (five seconds by default), bounded by what is left of the window. + /// regardless: half the relaxed timeout (five seconds by default), never more than half of what is left of the + /// window, never less than , and never more than the window. /// /// /// Two things pull in opposite directions. A blocking command in flight - BLPOP, XREAD BLOCK, WAIT - is failed @@ -522,14 +524,17 @@ private static TimeSpan Remaining(TimeSpan window, ValueStopwatch watch) /// because of the hold), and follows the one setting an application already uses to say how patient it is /// prepared to be during maintenance. The window still bounds it, because the server closes the socket when /// the window ends whatever we do: a command that cannot finish inside the window fails either way, and holding - /// past it would only delay everything else. A cap derived from the window alone would be wrong the other way: - /// a 15s window is not a reason to hold traffic for 15s. + /// past it would only delay everything else. Half of what is left of the window is an absolute ceiling as well, + /// whatever the relaxed timeout: a generous relaxed timeout must not turn the hold into a full-window stall that + /// recycles at the deadline with work in flight, which is the failure this whole mechanism exists to remove. /// private TimeSpan DrainCap(TimeSpan budget) { - var half = TimeSpan.FromTicks(Multiplexer.RawConfig.MaintenanceRelaxedTimeout.Ticks / 2); - if (half < DrainFloor) half = DrainFloor; // a relaxed timeout of zero disables relaxation; it must not disable the drain - return budget < half ? budget : half; + var cap = TimeSpan.FromTicks(Multiplexer.RawConfig.MaintenanceRelaxedTimeout.Ticks / 2); + var halfWindow = TimeSpan.FromTicks(budget.Ticks / 2); + if (cap > halfWindow) cap = halfWindow; + if (cap < DrainFloor) cap = DrainFloor; // a relaxed timeout of zero disables relaxation; it must not disable the drain + return cap > budget ? budget : cap; } /// The least the drain waits for in-flight commands, whatever the relaxed timeout: one round trip on any network, with room to spare. @@ -574,11 +579,14 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) // gets what little time remains. Otherwise the hold lasts up to the cap. var cap = budget < DrainFloor ? TimeSpan.Zero : DrainCap(budget); - // the held work, and any blocking command being waited for, must not be timed out by the hold itself, nor + // The held work, and any blocking command being waited for, must not be timed out by the hold itself, nor // by the reconnect that follows it; the relaxed window covers it where relaxation is on, and the cover below - // covers it regardless + // covers it regardless. The reconnect can take until the end of the window plus one connect attempt: a + // MoveTo pins the socket target to the named successor for the rest of the window, and a successor that + // does not answer is tried until that expires before the endpoint is resolved the ordinary way. ExtendRelaxedWindow(cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout, "MOVING handoff"); - SetHoldCover(cap + LockGrace + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout)); + var pinned = cap + LockGrace > budget ? cap + LockGrace : budget; + SetHoldCover(pinned + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout)); // snapshots: Dispose nulls the fields, and a handoff in progress must not fault on a server going away var interactiveBridge = interactive; diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index aa1de859e..817677011 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -905,11 +905,15 @@ await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8 // jitter is at most a tenth of the window and the drain is one round trip; recycling at the deadline is the bug Assert.True(recycledAfter < TimeSpan.FromSeconds(4), $"recycled after {recycledAfter}: that is waiting for the window, not draining"); - // the replacement carries the held work and whatever follows + // the replacement carries the held work and whatever follows. Note the in-process transport never dials the + // successor: its tunnel derives the socket target from the configured endpoint, so the reconnect goes + // straight back to this server; the timing here is the hold and the handshake, not a refused address. + var resumeWatch = System.Diagnostics.Stopwatch.StartNew(); var before = Volatile.Read(ref completed); Assert.True( - await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutMilliseconds: 10_000), + await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutMilliseconds: 20_000), "traffic should resume on the replacement connection"); + log.WriteLine($"traffic resumed {resumeWatch.ElapsedMilliseconds}ms after the recycle was observed"); stop.Cancel(); await Task.WhenAll(workers); From b94531f7a542a9b932255a1f4b579fbca779b55a Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 09:44:42 +0000 Subject: [PATCH 07/27] Handoff drain: held work survives the reconnect, per-bridge deadlines, cover ends on establishment, teardown outside the lock Sixth round of review follow-ups: - A handoff-pending marker, set alongside the hold cover and cleared when a connection is fully established (or by the cover's deadline), keeps a FailFast policy from abandoning held work when a connect attempt on the way to the replacement fails. The hold alone covered the hold, not the reconnect it exists for. - Each bridge is retired by its own loop with its own lock deadline, run side by side, so a writer wedged on one bridge's lock cannot spend the other's time and leave its idle connection to the server's close. - The hold cover is cleared when the interactive connection establishes; the deadline is only the bound for the case where it never does. A short configured timeout with relaxation off regains hang detection the moment the replacement is up, instead of after the whole window. - Under the write lock the retire now only marks the connection; the report and dispose follow the release, so event handlers and socket shutdown no longer stall every writer and the replacement's handshake. The retired flag on the connection is what keeps writers and the backlog processor off it in the gap. - The front queue's comments say what it orders and what it does not. - The RetiredForHandoff doc sits on its own member again. - The docs say the floor wins over the half-window bound under two seconds. - No kick helper: EndQuiesce and the two held-enqueue paths pair a full fence with their flag/queue accesses, so neither side misses the other and no processor is started for an empty backlog. - The cap tests assert against jitter-aware bounds. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 109 +++++++++++------- src/StackExchange.Redis/PhysicalConnection.cs | 18 +-- .../ServerEndPoint.Maintenance.cs | 94 ++++++++++----- src/StackExchange.Redis/ServerEndPoint.cs | 3 + .../MaintenanceNotificationTests.cs | 12 +- 6 files changed, 152 insertions(+), 86 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 5e642a25b..86aa639bd 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second, and never longer than the window itself, since the server closes the socket when that ends regardless. For the duration of the hold and the reconnect after it, commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment the replacement is up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index f428f6bbc..c9881eea4 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -37,8 +37,11 @@ internal sealed class PhysicalBridge : IDisposable /// /// Messages that go out before anything in : a writer that lost its connection to a - /// maintenance handoff while waiting for the write lock was issued before anything the hold has queued - /// since, so it is requeued here rather than appended. Every backlog operation drains this first. + /// maintenance handoff while waiting for the write lock is requeued here rather than appended, so that it + /// goes ahead of work the hold queued while it waited. This is a best effort at the ordering the writer + /// would have had, not a guarantee: a writer starved past the point where the replacement has begun + /// flushing held work lands behind what was already flushed, and nothing here orders a batch that straddles + /// the recycle - which is no more than the write path promised before. Every backlog operation drains this first. /// private readonly ConcurrentQueue _backlogFront = new(); @@ -143,6 +146,27 @@ public enum State : byte private volatile bool _quiescing; private PhysicalConnection? _quiesceTarget; private volatile bool _quiesceRecycled; + private int _handoffPendingDeadlineTicks; + + /// + /// Marks that a maintenance handoff is replacing this bridge's connection, for the given span at most, so that + /// work the hold diverted into the backlog survives the reconnect under a FailFast policy as well as the hold + /// itself: a connect attempt that fails on the way to the replacement is not a reason to abandon it. Cleared + /// when a connection is fully established, or by the deadline if none is. + /// + internal void SetHandoffPending(TimeSpan span) + => Volatile.Write(ref _handoffPendingDeadlineTicks, NonZero(unchecked(Environment.TickCount + (int)span.TotalMilliseconds))); + + private static int NonZero(int ticks) => ticks == 0 ? 1 : ticks; + + private bool HandoffPending + { + get + { + var deadline = Volatile.Read(ref _handoffPendingDeadlineTicks); + return deadline != 0 && unchecked(deadline - Environment.TickCount) > 0; + } + } /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has @@ -168,25 +192,12 @@ internal void EndQuiesce() if (!_quiescing) return; _quiescing = false; _quiesceTarget = null; - // Unconditionally, not only when the backlog has items: an enqueuer that read the flag just before it - // flipped did not kick, and neither side's volatile access orders its store before the other's load. A - // kick with nothing queued costs one idle pass of the processor. - KickBacklogProcessor(); - } - - /// Gets held or queued work moving: the processor may be lingering between passes, or may have exited. - private void KickBacklogProcessor() - { - try - { - _backlogAutoReset.Set(); - } - catch (ObjectDisposedException) - { - // disposed: nothing to wake, but fall through - a processor started now runs the disposed-drain that - // fails whatever is queued, which is the only thing that will, since the heartbeat has stopped - } - StartBacklogProcessor(); + // Store the flag, fence, then load the queue; an enqueuer stores the queue, fences, then loads the flag. + // With full fences on both sides at least one of them sees the other's store, so a message that lands + // as the hold is released is started by one side or the other, without spawning a processor for an + // empty backlog. StartBacklogProcessor wakes a processor that is already running. + Interlocked.MemoryBarrier(); + if (BacklogHasItems) StartBacklogProcessor(); } /// @@ -245,6 +256,7 @@ private static bool ShouldHoldBatch(List messages) var target = _quiesceTarget; if (target is null) return false; // nothing was connected when the hold began if (_quiesceRecycled) return true; + bool retire = false; bool gotLock; try @@ -272,17 +284,12 @@ private static bool ShouldHoldBatch(List messages) } if (!force && target.HasCallerMessagesAwaitingResponse()) return null; - Trace($"recycling {ConnectionType} connection: {reason}"); - - // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises - // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and - // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first - // also performs the disconnect, so the Dispose below is cleanup. - target.MarkRetiredForHandoff(); // a writer already past the hold check queues for the replacement instead (see TryRequeueForHandoff) + // Under the lock only the marking: from here no writer that takes the lock writes to the target (see + // TryRequeueForHandoff), and the backlog processor skips it too, so the teardown itself can happen + // after the release rather than stalling every writer behind event handlers and socket shutdown. + target.MarkRetiredForHandoff(); _quiesceRecycled = true; - target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); - target.Dispose(); - return true; + retire = true; } finally { @@ -295,6 +302,19 @@ private static bool ShouldHoldBatch(List messages) // disposed while we held it; nobody is waiting on a disposed mutex } } + + if (retire) + { + Trace($"recycling {ConnectionType} connection: {reason}"); + + // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises + // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and + // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first + // also performs the disconnect, so the Dispose below is cleanup. + target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + target.Dispose(); + } + return true; } public void Dispose() @@ -354,8 +374,13 @@ private WriteResult QueueOrFailMessage(Message message, bool holdForHandoff = fa message.SetEnqueued(null); BacklogEnqueue(message); // Note: we don't start a worker on each message here - except for held work that lands just after - // the hold was released, which nothing else would pick up until the heartbeat - if (holdForHandoff && !_quiescing) KickBacklogProcessor(); + // the hold was released, which nothing else would pick up until the heartbeat; see EndQuiesce for + // the fence pairing + if (holdForHandoff) + { + Interlocked.MemoryBarrier(); + if (!_quiescing) StartBacklogProcessor(); + } return WriteResult.Success; // Successfully queued, so indicate success } } @@ -651,7 +676,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur // is a replacement we chose, and not while a handoff is holding work back: that backlog is work diverted // off a live connection, which would have been written and answered, so a death of the old connection // during the hold (the window expiring, a blip) must leave it for the replacement to flush - if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !_quiescing) + if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !_quiescing && !HandoffPending) { AbandonPendingBacklog(innerException, connection); } @@ -752,6 +777,7 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) { reportNextFailure = reconfigureNextFailure = true; LastException = null; + Volatile.Write(ref _handoffPendingDeadlineTicks, 0); // the replacement is up; held work is flowing to it Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -1226,9 +1252,10 @@ private bool BacklogTryPeek([NotNullWhen(true)] out Message? message) => _backlogFront.TryPeek(out message) || _backlog.TryPeek(out message); /// - /// Requeues, ahead of everything else, a caller's message that lost its connection to a maintenance handoff - /// while waiting for the write lock: it was issued before anything the hold has queued since, and has not - /// been written. See . + /// Requeues, ahead of what is still queued, a caller's message that lost its connection to a maintenance + /// handoff while waiting for the write lock: it has not been written, and the work the hold queued while it + /// waited has not gone out either, so it goes first. See and the note on + /// about what this does and does not order. /// private void BacklogRequeueFront(Message message) { @@ -1244,8 +1271,8 @@ private void BacklogRequeueFront(Message message) } message.SetBacklogged(); if (wasEmpty) Multiplexer.Root(); // as BacklogEnqueue: a backlog with a caller's task in it must pin the multiplexer - // unconditionally: the hold may be released concurrently, and a kick with the hold still on costs one idle pass - KickBacklogProcessor(); + Interlocked.MemoryBarrier(); // paired with EndQuiesce: one side or the other sees that the hold is off and the queue is not empty + if (!_quiescing) StartBacklogProcessor(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -1518,7 +1545,7 @@ private void ProcessBridgeBacklog() // Snapshot the connection for the whole of this message: OnDisconnected nulls the field // without waiting for the write lock, so re-reading it (as the loop guard used to) can hand // null to a write path that requires a connection. See #3167. - if (this.physical is not { HasOutputPipe: true } physical) break; + if (this.physical is not { HasOutputPipe: true, RetiredForHandoff: false } physical) break; // retired by a handoff: its teardown follows the lock release Message? message; _backlogStatus = BacklogStatus.CheckingForWork; @@ -1646,7 +1673,7 @@ private WriteResult TimedOutBeforeWrite(Message message) /// private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, bool bypassBacklog, out WriteResult result) { - if (bypassBacklog || message.IsInternalCall || ReferenceEquals(physical, this.physical) || !physical.RetiredForHandoff) + if (bypassBacklog || message.IsInternalCall || !physical.RetiredForHandoff) { result = default; return false; diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 41047ba03..b7d964332 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -977,6 +977,15 @@ internal void GetStormLog(StringBuilder sb) } } + /// + /// Whether a maintenance handoff retired this connection, so that a writer which was headed for it and only + /// reached the write lock afterwards can tell a planned replacement from an ordinary closure. Set under the + /// bridge's write lock; the report and dispose follow once the lock is released. Never reset. + /// + internal bool RetiredForHandoff { get; private set; } + + internal void MarkRetiredForHandoff() => RetiredForHandoff = true; + /// /// Whether any message awaiting a response was issued by a *caller* rather than by us. /// @@ -990,15 +999,6 @@ internal void GetStormLog(StringBuilder sb) /// of walking a queue that a stalled server can leave thousands of entries long. /// /// - /// - /// Whether a maintenance handoff retired this connection, so that a writer which was headed for it and only - /// reached the write lock afterwards can tell a planned replacement from an ordinary closure. Set under the - /// bridge's write lock, immediately before the connection is disposed; never reset. - /// - internal bool RetiredForHandoff { get; private set; } - - internal void MarkRetiredForHandoff() => RetiredForHandoff = true; - internal bool HasCallerMessagesAwaitingResponse() { lock (_writtenAwaitingResponse) diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index a382eecff..5ad2b5ebb 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -233,6 +233,13 @@ private void SetHoldCover(TimeSpan span) Volatile.Write(ref _holdCoverDeadlineTicks, NudgeFromZero(unchecked(Environment.TickCount + ms))); } + /// + /// Ends the hold cover early: the interactive connection is fully established again, so the held work is + /// flowing and ordinary timeouts - including a deliberately short one, with relaxation turned off - apply once + /// more. The deadline set by is only the bound for the case where it never is. + /// + private void ClearHoldCover() => Volatile.Write(ref _holdCoverDeadlineTicks, 0); + /// /// Milliseconds of relaxation left, or zero if none; clears an expired window as a side-effect. /// @@ -581,68 +588,95 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) // The held work, and any blocking command being waited for, must not be timed out by the hold itself, nor // by the reconnect that follows it; the relaxed window covers it where relaxation is on, and the cover below - // covers it regardless. The reconnect can take until the end of the window plus one connect attempt: a - // MoveTo pins the socket target to the named successor for the rest of the window, and a successor that - // does not answer is tried until that expires before the endpoint is resolved the ordinary way. + // covers it regardless, until the interactive connection is established again or this span runs out. The + // reconnect can take until the end of the window plus one connect attempt: a MoveTo pins the socket target + // to the named successor for the rest of the window, and a successor that does not answer is tried until + // that expires before the endpoint is resolved the ordinary way. ExtendRelaxedWindow(cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout, "MOVING handoff"); var pinned = cap + LockGrace > budget ? cap + LockGrace : budget; - SetHoldCover(pinned + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout)); + var cover = pinned + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); + SetHoldCover(cover); // snapshots: Dispose nulls the fields, and a handoff in progress must not fault on a server going away var interactiveBridge = interactive; var subscriptionBridge = subscription; + + // held work must also survive the reconnect under a FailFast policy, for as long as the cover lasts or until + // the replacement is up, whichever is first + interactiveBridge?.SetHandoffPending(cover); + subscriptionBridge?.SetHandoffPending(cover); interactiveBridge?.BeginQuiesce(); subscriptionBridge?.BeginQuiesce(); + + // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's + // lock must not spend the other's time, and the other's idle connection must still be retired before the + // server closes it. Each bridge is released the moment its own connection is settled. + var interactiveTask = RetireBridgeAsync(interactiveBridge, watch, cap, reason); + var subscriptionTask = RetireBridgeAsync(subscriptionBridge, watch, cap, reason); + var interactiveResult = await interactiveTask.ForAwait(); + var subscriptionResult = await subscriptionTask.ForAwait(); + + var recycled = interactiveResult.Recycled == true || subscriptionResult.Recycled == true; + if (recycled) Interlocked.Increment(ref _handoffRecycles); + Multiplexer.Trace( + $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" + + (interactiveResult.Forced || subscriptionResult.Forced ? " (still busy at the drain cap)" : " (drained)") + + (interactiveResult.Recycled is null || subscriptionResult.Recycled is null ? " (write lock not obtained; left to the server)" : ""), + ToString()); + } + + /// + /// What did: whether the connection was recycled (null: the write lock + /// could not be had, and the connection was left to the server's close), and whether the cap was reached with + /// work still in flight. A struct rather than a tuple: the library does not reference System.ValueTuple. + /// + private readonly struct BridgeRetirement(bool? recycled, bool forced) + { + public bool? Recycled { get; } = recycled; + public bool Forced { get; } = forced; + } + + /// + /// Retires one bridge's connection once drained, or at the cap regardless, and releases its hold; see + /// . + /// + private async Task RetireBridgeAsync(PhysicalBridge? bridge, ValueStopwatch watch, TimeSpan cap, string reason) + { + if (bridge is null) return new(false, false); + bool forced = false; try { using var lockDeadline = new CancellationTokenSource(cap + LockGrace); - bool? interactiveResult = interactiveBridge is null ? false : null; - bool? subscriptionResult = subscriptionBridge is null ? false : null; - bool forced = false; while (true) { var force = watch.ElapsedMilliseconds >= cap.TotalMilliseconds; forced |= force; - // each bridge is released the moment its own connection is settled: the interactive bridge usually - // drains in a round trip, and must not keep holding work while the subscription bridge takes longer - if (interactiveResult is null && (interactiveResult = await interactiveBridge!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait()) is not null) Release(interactiveBridge); - if (subscriptionResult is null && (subscriptionResult = await subscriptionBridge!.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait()) is not null) Release(subscriptionBridge); - if (interactiveResult is not null && subscriptionResult is not null) break; + var result = await bridge.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); + if (result is { } done) return new(done, forced); if (lockDeadline.IsCancellationRequested) { // the write lock could not be had even for the forced pass: something is stuck on it. Recycling // without the lock would reopen the mid-write loss the lock exists to prevent, so the connection // is left to the server's close, which is where things stood before handoffs existed - break; + return new(null, forced); } await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } - - var recycled = interactiveResult == true || subscriptionResult == true; - if (recycled) Interlocked.Increment(ref _handoffRecycles); - Multiplexer.Trace( - $"MOVING: {(recycled ? "recycled" : "nothing to recycle")} after {watch.ElapsedMilliseconds}ms" - + (forced ? " (still busy at the drain cap)" : " (drained)") - + (interactiveResult is null || subscriptionResult is null ? " (write lock not obtained; left to the server)" : ""), - ToString()); } - finally + catch (Exception ex) { - // whatever was not released in the loop (the lock-not-obtained exit, an exception): EndQuiesce is idempotent - Release(interactiveBridge); - Release(subscriptionBridge); + Multiplexer.OnInternalError(ex, EndPoint); + return new(null, forced); } - - void Release(PhysicalBridge? bridge) + finally { - if (bridge is null) return; try { - bridge.EndQuiesce(); + bridge.EndQuiesce(); // idempotent; one bridge failing to release must not affect the other } catch (Exception ex) { - Multiplexer.OnInternalError(ex, EndPoint); // one bridge failing to release must not leave the other holding work + Multiplexer.OnInternalError(ex, EndPoint); } } } diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index e203139c6..0b675f96a 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1129,6 +1129,9 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // whatever a handoff pointed us at, we are connected now: resume normal resolution ClearHandoffTarget(); + // and the hold cover has done its job once the connection that carries commands is up again + if (bridge.ConnectionType == ConnectionType.Interactive) ClearHoldCover(); + // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; // and if so, is pub/sub sharing it? (if not, subscriptions live on a dedicated connection) diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 817677011..ffc9c309b 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -959,7 +959,8 @@ await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 1 // the slow command was answered, not failed: the recycle waited for it foreach (var pending in slow) Assert.True(await pending >= TimeSpan.Zero); Assert.True(recycledAfter >= TimeSpan.FromSeconds(2), $"recycled after {recycledAfter}: before the in-flight command could have been answered"); - Assert.True(recycledAfter < TimeSpan.FromSeconds(6), $"recycled after {recycledAfter}: later than the cap plus jitter"); + // 3s for the command, up to 1s of jitter, lock and poll granularity, and slack; the cap would be 5s + jitter + Assert.True(recycledAfter < TimeSpan.FromSeconds(7), $"recycled after {recycledAfter}: later than the command plus jitter"); } } @@ -974,7 +975,7 @@ public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() using (server) await using (conn) { - server.SetLatency(TimeSpan.FromSeconds(9)); // longer than the 5s cap; shorter than the relaxed 10s timeout + server.SetLatency(TimeSpan.FromSeconds(12)); // well past the 5s cap and the slack below; the relaxed window covers it until the recycle var db = conn.GetDatabase(); var blocking = Enumerable.Range(0, 3).Select(_ => db.PingAsync()).ToArray(); await Task.Delay(250); // let them be written @@ -991,10 +992,11 @@ await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 1 "the handoff should have replaced the connection"); var recycledAfter = watch.Elapsed; server.SetLatency(TimeSpan.Zero); // so the replacement's handshake is not slowed too - log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with 9s commands in flight"); + log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with 12s commands in flight"); - // the cap (5s) plus at most a second of jitter: well short of the 9s the commands need - Assert.True(recycledAfter < TimeSpan.FromSeconds(7), $"recycled after {recycledAfter}: the drain waited for the blocking commands instead of capping"); + // jitter (at most a tenth of the window, capped at 1s) + the 5s cap + the lock grace (1s) + poll granularity, + // with a further second of slack: still well short of the 12s the commands need + Assert.True(recycledAfter < TimeSpan.FromSeconds(9), $"recycled after {recycledAfter}: the drain waited for the blocking commands instead of capping"); // the in-flight commands were failed by the recycle, which is what the server's close would have done foreach (var pending in blocking) From 0fc57a4e62b599540e42bbcd0854e6447fbd0787 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 10:24:12 +0000 Subject: [PATCH 08/27] Handoff drain: cover ends when both bridges are up, no writer touches a retired connection, FailFast exemption scoped, recycle path retires what is current Seventh round of review follow-ups: - The hold cover is cleared when both the interactive and the subscription connection are established (or the subscription is not required), not when the interactive one alone is: held subscribes must not be swept while the subscription bridge is still reconnecting. - A writer that reaches the lock with a retired snapshot never writes to it, internal and backlog-bypassing messages included: those fail as having no connection instead of faulting the write mid-teardown and tearing down the bridge; a caller's message is requeued as before. - The handoff-pending exemption from a FailFast abandon applies only to the handoff's own failures: the retired connection, or a replacement attempt that never got as far as a working connection. An established replacement that then fails is an ordinary failure. The marker is cleared when the handoff gives up on a bridge or finds nothing of its own to replace. - On the no-successor paths, a target that went away by itself during the hold is replaced by whatever is current, since the ordinary reconnect may have landed back on the node being retired; with a successor pinned the reconnect owns what happens next, as before. - The handoff tests gate on the slow command being observed in flight rather than on a fixed delay, and their time bounds carry more slack. - The docs say "approximately in the order they were issued". - One tick-sentinel helper (ServerEndPoint.NudgeFromZeroTicks) and one scan over both backlog queues. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 106 ++++++++++++------ .../ServerEndPoint.Maintenance.cs | 49 +++++--- src/StackExchange.Redis/ServerEndPoint.cs | 8 +- .../MaintenanceNotificationTests.cs | 12 +- 5 files changed, 112 insertions(+), 65 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 86aa639bd..1b3a922a1 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, in the order they were issued. This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment the replacement is up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment the replacement is up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index c9881eea4..a32b80628 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -155,9 +155,10 @@ public enum State : byte /// when a connection is fully established, or by the deadline if none is. /// internal void SetHandoffPending(TimeSpan span) - => Volatile.Write(ref _handoffPendingDeadlineTicks, NonZero(unchecked(Environment.TickCount + (int)span.TotalMilliseconds))); + => Volatile.Write(ref _handoffPendingDeadlineTicks, ServerEndPoint.NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)span.TotalMilliseconds))); - private static int NonZero(int ticks) => ticks == 0 ? 1 : ticks; + /// The handoff gave up, or found nothing of its own to replace: the pending marker no longer applies. + internal void ClearHandoffPending() => Volatile.Write(ref _handoffPendingDeadlineTicks, 0); private bool HandoffPending { @@ -168,6 +169,21 @@ private bool HandoffPending } } + /// + /// Whether a connection failure belongs to a maintenance handoff in progress: the connection it retired, or an + /// attempt to bring up the replacement that did not get as far as a working connection. An established + /// replacement that then fails is an ordinary failure, and a FailFast policy applies to it as usual, even + /// while the handoff's pending marker is still set. + /// + private bool IsHandoffOwnFailure(PhysicalConnection connection, ConnectionFailureType failureType) + => HandoffPending + && (connection.RetiredForHandoff + || failureType is ConnectionFailureType.UnableToConnect + or ConnectionFailureType.UnableToResolvePhysicalConnection + or ConnectionFailureType.AuthenticationFailure + or ConnectionFailureType.ProtocolFailure + or ConnectionFailureType.Loading); + /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has /// already been written complete; see . The connection @@ -245,13 +261,16 @@ private static bool ShouldHoldBatch(List messages) /// queues for the replacement instead (see ). /// /// Replace the connection even if caller work is still in flight on it. + /// When the connection the hold began on has gone away by itself, replace + /// whatever is current instead of declaring there is nothing to do: with no successor to pin the reconnect to, + /// the ordinary reconnect may have landed straight back on the node being retired. /// Why; for the trace. /// Bounds the wait for the write lock (together with the mutex's own timeout, whichever /// ends first); the drain re-waits until this deadline, so it is the deadline that decides how long overall. /// null if the connection could not be retired yet - caller work in flight, or the lock not /// obtained inside the deadline - so the drain should try again; otherwise whether this call, or an earlier /// one, replaced it. - internal async ValueTask TryRetireQuiesceTargetAsync(bool force, string reason, CancellationToken lockDeadline) + internal async ValueTask TryRetireQuiesceTargetAsync(bool force, bool retireCurrentIfTargetGone, string reason, CancellationToken lockDeadline) { var target = _quiesceTarget; if (target is null) return false; // nothing was connected when the hold began @@ -275,12 +294,20 @@ private static bool ShouldHoldBatch(List messages) try { if (_quiesceRecycled) return true; - if (!ReferenceEquals(physical, target)) + var current = physical; + if (!ReferenceEquals(current, target)) { - // it went away by itself (the server closed it, or it faulted) and the ordinary reconnect owns - // what happens next; there is nothing left for the handoff to retire - _quiesceRecycled = true; - return false; + if (!retireCurrentIfTargetGone || current is null) + { + // it went away by itself (the server closed it, or it faulted) and the reconnect, pinned to + // the named successor, owns what happens next; there is nothing left for the handoff to retire + _quiesceRecycled = true; + return false; + } + // it went away by itself and the ordinary reconnect, with no successor to pin to, may well have + // landed back on the node being retired: what is current now is what the handoff replaces + target = current; + _quiesceTarget = current; } if (!force && target.HasCallerMessagesAwaitingResponse()) return null; @@ -676,7 +703,8 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur // is a replacement we chose, and not while a handoff is holding work back: that backlog is work diverted // off a live connection, which would have been written and answered, so a death of the old connection // during the hold (the window expiring, a blip) must leave it for the replacement to flush - if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !_quiescing && !HandoffPending) + if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !_quiescing + && !IsHandoffOwnFailure(connection, failureType)) { AbandonPendingBacklog(innerException, connection); } @@ -1251,6 +1279,20 @@ private bool BacklogTryDequeue([NotNullWhen(true)] out Message? message) private bool BacklogTryPeek([NotNullWhen(true)] out Message? message) => _backlogFront.TryPeek(out message) || _backlog.TryPeek(out message); + /// Whether either queue holds a caller's message; snapshot enumeration, which a concurrent queue allows. + private bool AnyCallerFacingQueued() + { + foreach (var message in _backlogFront) + { + if (message.IsCallerFacing) return true; + } + foreach (var message in _backlog) + { + if (message.IsCallerFacing) return true; + } + return false; + } + /// /// Requeues, ahead of what is still queued, a caller's message that lost its connection to a maintenance /// handoff while waiting for the write lock: it has not been written, and the work the hold queued while it @@ -1335,15 +1377,7 @@ internal bool HasCallerWork() { if (physical?.HasCallerMessagesAwaitingResponse() == true) return true; - foreach (var message in _backlogFront) // snapshot enumeration; a concurrent queue is fine to walk - { - if (message.IsCallerFacing) return true; - } - foreach (var message in _backlog) - { - if (message.IsCallerFacing) return true; - } - return false; + return AnyCallerFacingQueued(); } /// @@ -1639,17 +1673,7 @@ internal void TryFlushStagedWrites(PhysicalConnection physical) public bool HasPendingCallerFacingItems() { - if (BacklogHasItems) - { - foreach (var item in _backlogFront) // non-consuming, thread-safe, etc - { - if (item.IsCallerFacing) return true; - } - foreach (var item in _backlog) - { - if (item.IsCallerFacing) return true; - } - } + if (BacklogHasItems && AnyCallerFacingQueued()) return true; return physical?.HasPendingCallerFacingItems() ?? false; } @@ -1663,21 +1687,29 @@ private WriteResult TimedOutBeforeWrite(Message message) } /// - /// For a writer that has just taken the write lock: if the connection it was headed for was replaced by a - /// maintenance handoff while it waited, the message - not yet written - is queued for the replacement at the - /// head of the backlog, since it was issued before anything the hold has queued since. The connection itself - /// says whether a handoff retired it (), so a writer starved - /// across any number of later handoffs still recognises its own. Only the handoff case: a write that loses its - /// connection to an ordinary closure is still reported as a closure (#3167), a message bypassing the backlog is - /// never queued, and internal traffic is exempt because the handshake writes to a connection before it is current. + /// For a writer that has just taken the write lock: if the connection it was headed for was retired by a + /// maintenance handoff while it waited, nothing is written to it - its teardown follows the lock release. A + /// caller's message is queued for the replacement at the head of the backlog, since it was issued before + /// anything the hold has queued since; internal traffic and a message bypassing the backlog are failed as + /// having no connection, as they would be on a disconnected bridge. The connection itself says whether a + /// handoff retired it (), so a writer starved across any + /// number of later handoffs still recognises its own. Only the handoff case: a write that loses its connection + /// to an ordinary closure is still reported as a closure (#3167). /// private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, bool bypassBacklog, out WriteResult result) { - if (bypassBacklog || message.IsInternalCall || !physical.RetiredForHandoff) + if (!physical.RetiredForHandoff) { result = default; return false; } + if (bypassBacklog || message.IsInternalCall) + { + // not queued, but not written to a connection whose teardown is under way either: a keep-alive or a + // backlog-bypassing message that reached the lock after the retire simply has no connection + result = FailDueToNoConnection(message); + return true; + } BacklogRequeueFront(message); result = WriteResult.Success; // queued counts as success return true; diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 5ad2b5ebb..954f30b13 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -230,13 +230,13 @@ private void SetHoldCover(TimeSpan span) { var ms = (int)span.TotalMilliseconds; Volatile.Write(ref _holdCoverMilliseconds, ms); - Volatile.Write(ref _holdCoverDeadlineTicks, NudgeFromZero(unchecked(Environment.TickCount + ms))); + Volatile.Write(ref _holdCoverDeadlineTicks, NudgeFromZeroTicks(unchecked(Environment.TickCount + ms))); } /// - /// Ends the hold cover early: the interactive connection is fully established again, so the held work is - /// flowing and ordinary timeouts - including a deliberately short one, with relaxation turned off - apply once - /// more. The deadline set by is only the bound for the case where it never is. + /// Ends the hold cover early: both connections are fully established again, so the held work is flowing and + /// ordinary timeouts - including a deliberately short one, with relaxation turned off - apply once more. The + /// deadline set by is only the bound for the case where that never happens. /// private void ClearHoldCover() => Volatile.Write(ref _holdCoverDeadlineTicks, 0); @@ -321,12 +321,12 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? if (tail <= TimeSpan.Zero) { Volatile.Write(ref _relaxedDeadlineTicks, 0); - Volatile.Write(ref _relaxedEndedTicks, NudgeFromZero(Environment.TickCount)); + Volatile.Write(ref _relaxedEndedTicks, NudgeFromZeroTicks(Environment.TickCount)); Multiplexer.Trace($"{type}: relaxation ended", ToString()); return; } - var deadline = NudgeFromZero(unchecked(Environment.TickCount + (int)tail.TotalMilliseconds)); + var deadline = NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)tail.TotalMilliseconds)); Volatile.Write(ref _relaxedDeadlineTicks, deadline); Volatile.Write(ref _relaxedEndedTicks, deadline); Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); @@ -472,7 +472,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? switch (decision.Action) { case HandoffAction.Recycle: - await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: false).ForAwait(); replaced = true; break; case HandoffAction.RecycleAtHalfWindow: @@ -480,7 +480,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // go on. Half of the *announced* window, less whatever the jitter already spent. var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); - await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason).ForAwait(); + await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successorPinned: false).ForAwait(); replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: @@ -489,7 +489,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining); - await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: true).ForAwait(); replaced = true; break; default: @@ -578,7 +578,12 @@ private TimeSpan DrainCap(TimeSpan budget) /// were warned. /// /// - private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) + /// What is left of the announced window. + /// Why; for the trace. + /// Whether reconnection is pinned to a named successor. If not, a connection that + /// went away by itself during the hold may have been replaced by one on the very node being retired, and that + /// replacement is retired instead; see . + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool successorPinned) { var watch = ValueStopwatch.StartNew(); // With less than the floor left of the window there is no hold worth having: the server is about to close @@ -611,8 +616,8 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the // server closes it. Each bridge is released the moment its own connection is settled. - var interactiveTask = RetireBridgeAsync(interactiveBridge, watch, cap, reason); - var subscriptionTask = RetireBridgeAsync(subscriptionBridge, watch, cap, reason); + var interactiveTask = RetireBridgeAsync(interactiveBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned); + var subscriptionTask = RetireBridgeAsync(subscriptionBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned); var interactiveResult = await interactiveTask.ForAwait(); var subscriptionResult = await subscriptionTask.ForAwait(); @@ -640,7 +645,7 @@ private readonly struct BridgeRetirement(bool? recycled, bool forced) /// Retires one bridge's connection once drained, or at the cap regardless, and releases its hold; see /// . /// - private async Task RetireBridgeAsync(PhysicalBridge? bridge, ValueStopwatch watch, TimeSpan cap, string reason) + private async Task RetireBridgeAsync(PhysicalBridge? bridge, ValueStopwatch watch, TimeSpan cap, string reason, bool retireCurrentIfTargetGone) { if (bridge is null) return new(false, false); bool forced = false; @@ -651,13 +656,21 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V { var force = watch.ElapsedMilliseconds >= cap.TotalMilliseconds; forced |= force; - var result = await bridge.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); - if (result is { } done) return new(done, forced); + var result = await bridge.TryRetireQuiesceTargetAsync(force, retireCurrentIfTargetGone, reason, lockDeadline.Token).ForAwait(); + if (result is { } done) + { + // nothing of the handoff's own is pending if nothing was replaced: the pending marker must not + // then shield a coincidental failure from a FailFast policy + if (!done) bridge.ClearHandoffPending(); + return new(done, forced); + } if (lockDeadline.IsCancellationRequested) { // the write lock could not be had even for the forced pass: something is stuck on it. Recycling // without the lock would reopen the mid-write loss the lock exists to prevent, so the connection - // is left to the server's close, which is where things stood before handoffs existed + // is left to the server's close, which is where things stood before handoffs existed - and the + // handoff has given up on this bridge, so its pending marker goes too + bridge.ClearHandoffPending(); return new(null, forced); } await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); @@ -666,6 +679,7 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V catch (Exception ex) { Multiplexer.OnInternalError(ex, EndPoint); + bridge.ClearHandoffPending(); return new(null, forced); } finally @@ -726,7 +740,7 @@ private static Random RandomFor(ServerEndPoint server) => private void ExtendRelaxedWindow(TimeSpan duration, string cause) { - var candidate = NudgeFromZero(unchecked(Environment.TickCount + (int)duration.TotalMilliseconds)); + var candidate = NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)duration.TotalMilliseconds)); while (true) { var current = Volatile.Read(ref _relaxedDeadlineTicks); @@ -797,7 +811,6 @@ internal MaintenanceNotificationType GetMaintenanceTypeForFault(int timeoutMilli /// /// Zero is the "no window" sentinel, so a deadline that lands on it moves by a tick. /// - private static int NudgeFromZero(int ticks) => ticks == 0 ? 1 : ticks; /// /// Whether this notification is new, per the conservative dedup described on . diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 0b675f96a..8cff10e84 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1091,7 +1091,8 @@ internal void OnRepeatedConnectFailure(int consecutiveFailures) } /// Zero means "never", so a tick count that lands on it moves by one. - private static int NudgeFromZeroTicks(int ticks) => ticks == 0 ? 1 : ticks; + /// Zero is the "unset" sentinel for tick-count fields; a genuine zero tick count is nudged to one. + internal static int NudgeFromZeroTicks(int ticks) => ticks == 0 ? 1 : ticks; /// /// Subscribes to the configuration-change broadcast on a RESP3 interactive connection that is shared with @@ -1129,8 +1130,9 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // whatever a handoff pointed us at, we are connected now: resume normal resolution ClearHandoffTarget(); - // and the hold cover has done its job once the connection that carries commands is up again - if (bridge.ConnectionType == ConnectionType.Interactive) ClearHoldCover(); + // and the hold cover has done its job once both connections are up again: the subscription bridge + // holds caller work too (subscribes), and must not be timed out while it is still reconnecting + if (IsConnected && IsSubscriberConnectedOrNotRequired) ClearHoldCover(); // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index ffc9c309b..788c5b811 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -903,7 +903,7 @@ await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8 log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms of a 10s window"); // jitter is at most a tenth of the window and the drain is one round trip; recycling at the deadline is the bug - Assert.True(recycledAfter < TimeSpan.FromSeconds(4), $"recycled after {recycledAfter}: that is waiting for the window, not draining"); + Assert.True(recycledAfter < TimeSpan.FromSeconds(5), $"recycled after {recycledAfter}: that is waiting for the window, not draining"); // the replacement carries the held work and whatever follows. Note the in-process transport never dials the // successor: its tunnel derives the socket target from the configured endpoint, so the reconnect goes @@ -940,7 +940,7 @@ public async Task MovingWaitsForSlowCommandsThatFitInsideTheDrainCap() server.SetLatency(TimeSpan.FromSeconds(3)); // the in-process server answers one request at a time, so one slow command, not several var db = conn.GetDatabase(); var slow = new[] { db.PingAsync() }; - await Task.Delay(250); // let it be written + Assert.True(await Poll.UntilAsync(() => conn.GetCounters().Interactive.SentItemsAwaitingResponse > 0), "the slow command should be in flight before MOVING arrives"); var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); @@ -956,9 +956,9 @@ await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 1 server.SetLatency(TimeSpan.Zero); log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with a 3s command in flight"); - // the slow command was answered, not failed: the recycle waited for it + // the slow command was answered, not failed: the recycle waited for it (had it not, the recycle would have + // failed the command, and this await would throw) foreach (var pending in slow) Assert.True(await pending >= TimeSpan.Zero); - Assert.True(recycledAfter >= TimeSpan.FromSeconds(2), $"recycled after {recycledAfter}: before the in-flight command could have been answered"); // 3s for the command, up to 1s of jitter, lock and poll granularity, and slack; the cap would be 5s + jitter Assert.True(recycledAfter < TimeSpan.FromSeconds(7), $"recycled after {recycledAfter}: later than the command plus jitter"); } @@ -978,7 +978,7 @@ public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() server.SetLatency(TimeSpan.FromSeconds(12)); // well past the 5s cap and the slack below; the relaxed window covers it until the recycle var db = conn.GetDatabase(); var blocking = Enumerable.Range(0, 3).Select(_ => db.PingAsync()).ToArray(); - await Task.Delay(250); // let them be written + Assert.True(await Poll.UntilAsync(() => conn.GetCounters().Interactive.SentItemsAwaitingResponse > 0), "the blocking commands should be in flight before MOVING arrives"); var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); @@ -996,7 +996,7 @@ await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 1 // jitter (at most a tenth of the window, capped at 1s) + the 5s cap + the lock grace (1s) + poll granularity, // with a further second of slack: still well short of the 12s the commands need - Assert.True(recycledAfter < TimeSpan.FromSeconds(9), $"recycled after {recycledAfter}: the drain waited for the blocking commands instead of capping"); + Assert.True(recycledAfter < TimeSpan.FromSeconds(10), $"recycled after {recycledAfter}: the drain waited for the blocking commands instead of capping"); // the in-flight commands were failed by the recycle, which is what the server's close would have done foreach (var pending in blocking) From 77e01b801a4f6ab00c813bd5a559442df9b73c2f Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 10:51:14 +0000 Subject: [PATCH 09/27] Handoff drain: one lifecycle for the FailFast shield and the timeout cover Eighth round of review follow-ups. The protection a handoff raises - the per-bridge shield against a FailFast policy abandoning held work, and the server's floor on the effective timeout - now shares a single lifecycle: raised in BeginQuiesce for a stated span, lowered only when a replacement connection fully establishes on that bridge, when the handoff gives up on it, or when the span runs out. The server's cover follows the bridges: it stands down when no bridge still has its protection up. - A target that went away by itself during the hold no longer stands the shield down: a replacement reconnect is in flight, and its first failure would otherwise have abandoned the held backlog. - The shield covers failures of the retired connection and of any connection that never completed its handshake, whatever the failure type; a reset mid-handshake reports as a socket failure like any other. A replacement that was fully established and then failed is ordinary. - The cover is no longer cleared by the old subscription connection still being live while the interactive replacement comes up; it waits for every bridge that still has a retire or reconnect pending. - When nothing recycles and nothing re-establishes, the cover stands down with the shield rather than waiting for its own deadline. - The orphaned sentinel doc and the doubled summary are gone; the docs say that under a second of window there is no hold at all. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 52 ++++++++++--------- src/StackExchange.Redis/PhysicalConnection.cs | 9 ++++ .../ServerEndPoint.Maintenance.cs | 32 +++++++----- src/StackExchange.Redis/ServerEndPoint.cs | 7 ++- 5 files changed, 60 insertions(+), 42 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 1b3a922a1..2f9035deb 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment the replacement is up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment the replacement is up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index a32b80628..afd473750 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -146,21 +146,24 @@ public enum State : byte private volatile bool _quiescing; private PhysicalConnection? _quiesceTarget; private volatile bool _quiesceRecycled; - private int _handoffPendingDeadlineTicks; /// - /// Marks that a maintenance handoff is replacing this bridge's connection, for the given span at most, so that - /// work the hold diverted into the backlog survives the reconnect under a FailFast policy as well as the hold - /// itself: a connect attempt that fails on the way to the replacement is not a reason to abandon it. Cleared - /// when a connection is fully established, or by the deadline if none is. + /// While set, a maintenance handoff is replacing this bridge's connection, and work the hold diverted into the + /// backlog survives the reconnect under a FailFast policy as well as the hold itself: a failure on the way to + /// the replacement is not a reason to abandon it. Raised in ; lowered when a + /// replacement is fully established, when the handoff gives up on this bridge, or by the deadline. The server's + /// timeout cover shares exactly this lifecycle, across both bridges. /// - internal void SetHandoffPending(TimeSpan span) - => Volatile.Write(ref _handoffPendingDeadlineTicks, ServerEndPoint.NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)span.TotalMilliseconds))); + private int _handoffPendingDeadlineTicks; - /// The handoff gave up, or found nothing of its own to replace: the pending marker no longer applies. - internal void ClearHandoffPending() => Volatile.Write(ref _handoffPendingDeadlineTicks, 0); + /// The handoff has given up on this bridge: its protection stands down. + internal void ClearHandoffPending() + { + Volatile.Write(ref _handoffPendingDeadlineTicks, 0); + ServerEndPoint.OnHandoffProtectionLowered(); + } - private bool HandoffPending + internal bool HandoffPending { get { @@ -171,18 +174,13 @@ private bool HandoffPending /// /// Whether a connection failure belongs to a maintenance handoff in progress: the connection it retired, or an - /// attempt to bring up the replacement that did not get as far as a working connection. An established - /// replacement that then fails is an ordinary failure, and a FailFast policy applies to it as usual, even - /// while the handoff's pending marker is still set. + /// attempt to bring up the replacement that never completed its handshake - whatever the failure type, since + /// a reset mid-handshake reports as a socket failure like any other. A replacement that was fully established + /// and then failed is an ordinary failure, and a FailFast policy applies to it as usual, even while the + /// handoff's protection is still up. /// - private bool IsHandoffOwnFailure(PhysicalConnection connection, ConnectionFailureType failureType) - => HandoffPending - && (connection.RetiredForHandoff - || failureType is ConnectionFailureType.UnableToConnect - or ConnectionFailureType.UnableToResolvePhysicalConnection - or ConnectionFailureType.AuthenticationFailure - or ConnectionFailureType.ProtocolFailure - or ConnectionFailureType.Loading); + private bool IsHandoffOwnFailure(PhysicalConnection connection) + => HandoffPending && (connection.RetiredForHandoff || !connection.EverFullyEstablished); /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has @@ -196,10 +194,13 @@ or ConnectionFailureType.ProtocolFailure /// something in flight, so the recycle waited for the end of the announced window and then failed whatever /// was in flight at that instant. /// - internal void BeginQuiesce() + /// For how long at most the handoff's protection stands - the FailFast shield here and + /// the timeout cover on the server - if no replacement establishes first. + internal void BeginQuiesce(TimeSpan protection) { _quiesceTarget = physical; _quiesceRecycled = false; + Volatile.Write(ref _handoffPendingDeadlineTicks, ServerEndPoint.NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)protection.TotalMilliseconds))); _quiescing = true; } @@ -704,7 +705,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur // off a live connection, which would have been written and answered, so a death of the old connection // during the hold (the window expiring, a blip) must leave it for the replacement to flush if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !_quiescing - && !IsHandoffOwnFailure(connection, failureType)) + && !IsHandoffOwnFailure(connection)) { AbandonPendingBacklog(innerException, connection); } @@ -800,12 +801,15 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) Trace("OnFullyEstablished"); // the attempt itself succeeded (handshake complete), even if this bridge has since moved on and discards it below connection.ReportAttemptOutcome(isSuccess: true, ConnectionFailureType.None, null); + connection.MarkFullyEstablished(); connection.SetIdle(); if (physical == connection && !isDisposed && ChangeState(State.ConnectedEstablishing, State.ConnectedEstablished)) { reportNextFailure = reconfigureNextFailure = true; LastException = null; - Volatile.Write(ref _handoffPendingDeadlineTicks, 0); // the replacement is up; held work is flowing to it + // a connection establishing after the hold began is the handoff's replacement: the protection stands + // down for this bridge (ServerEndPoint.OnFullyEstablished lowers the shared cover once no bridge needs it) + Volatile.Write(ref _handoffPendingDeadlineTicks, 0); Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index b7d964332..e45d4b261 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -986,6 +986,15 @@ internal void GetStormLog(StringBuilder sb) internal void MarkRetiredForHandoff() => RetiredForHandoff = true; + /// + /// Whether this connection ever completed its handshake. A failure on a connection that never did is a failed + /// attempt to bring a connection up, which is how a maintenance handoff tells "the replacement is not up yet" + /// from "the replacement was up and then failed" without enumerating failure types. + /// + internal bool EverFullyEstablished { get; private set; } + + internal void MarkFullyEstablished() => EverFullyEstablished = true; + /// /// Whether any message awaiting a response was issued by a *caller* rather than by us. /// diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 954f30b13..022152e1d 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -240,6 +240,18 @@ private void SetHoldCover(TimeSpan span) /// private void ClearHoldCover() => Volatile.Write(ref _holdCoverDeadlineTicks, 0); + /// Whether either bridge still has a handoff's protection up; see . + internal bool AnyHandoffPending => interactive?.HandoffPending == true || subscription?.HandoffPending == true; + + /// + /// A bridge's handoff protection has just stood down (its replacement established, or the handoff gave up on it); + /// the server's timeout cover follows the same lifecycle and stands down once no bridge needs it. + /// + internal void OnHandoffProtectionLowered() + { + if (!AnyHandoffPending) ClearHoldCover(); + } + /// /// Milliseconds of relaxation left, or zero if none; clears an expired window as a side-effect. /// @@ -606,12 +618,11 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su var interactiveBridge = interactive; var subscriptionBridge = subscription; - // held work must also survive the reconnect under a FailFast policy, for as long as the cover lasts or until - // the replacement is up, whichever is first - interactiveBridge?.SetHandoffPending(cover); - subscriptionBridge?.SetHandoffPending(cover); - interactiveBridge?.BeginQuiesce(); - subscriptionBridge?.BeginQuiesce(); + // one lifecycle for the protection: raised here with the hold, on each bridge (the FailFast shield) and on the + // server (the timeout cover); lowered only when a replacement establishes, when the handoff gives up on a + // bridge, or when this span runs out - never on a local idea of "done" along the way + interactiveBridge?.BeginQuiesce(cover); + subscriptionBridge?.BeginQuiesce(cover); // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the @@ -659,9 +670,8 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V var result = await bridge.TryRetireQuiesceTargetAsync(force, retireCurrentIfTargetGone, reason, lockDeadline.Token).ForAwait(); if (result is { } done) { - // nothing of the handoff's own is pending if nothing was replaced: the pending marker must not - // then shield a coincidental failure from a FailFast policy - if (!done) bridge.ClearHandoffPending(); + // whether we replaced the connection or it went away by itself, a replacement is now on its way, + // and the protection stays up for it until it establishes or the span runs out return new(done, forced); } if (lockDeadline.IsCancellationRequested) @@ -808,10 +818,6 @@ internal MaintenanceNotificationType GetMaintenanceTypeForFault(int timeoutMilli : MaintenanceNotificationType.None; } - /// - /// Zero is the "no window" sentinel, so a deadline that lands on it moves by a tick. - /// - /// /// Whether this notification is new, per the conservative dedup described on . /// diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 8cff10e84..b74b81fde 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1091,7 +1091,6 @@ internal void OnRepeatedConnectFailure(int consecutiveFailures) } /// Zero means "never", so a tick count that lands on it moves by one. - /// Zero is the "unset" sentinel for tick-count fields; a genuine zero tick count is nudged to one. internal static int NudgeFromZeroTicks(int ticks) => ticks == 0 ? 1 : ticks; /// @@ -1130,9 +1129,9 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // whatever a handoff pointed us at, we are connected now: resume normal resolution ClearHandoffTarget(); - // and the hold cover has done its job once both connections are up again: the subscription bridge - // holds caller work too (subscribes), and must not be timed out while it is still reconnecting - if (IsConnected && IsSubscriberConnectedOrNotRequired) ClearHoldCover(); + // and a handoff's timeout cover stands down with the last bridge that still needed it: a bridge + // that has not yet retired its connection, or is still reconnecting, keeps it up + OnHandoffProtectionLowered(); // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; From a5d48048c9d5669f907af32463c375493d10b45c Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 11:24:21 +0000 Subject: [PATCH 10/27] Handoff drain: drop the front queue, lower protection on an empty bridge, leave a replacement that landed elsewhere alone Ninth round of review follow-ups: - The second backlog queue is gone. A writer that lost its connection to the handoff is appended to the backlog like the held work; the head insert bought only a best effort at ordering, at the price of a head-mutation discipline across every backlog operation. The abandon path keeps the dequeuers' lock, which closes a pre-existing race with the timeout sweep. - A bridge with no connection when the hold began lowers its protection at once, so a dead subscription bridge cannot keep the timeout floor up for the whole cover span while the interactive replacement is serving. - On the no-successor paths, a replacement that demonstrably landed on a different node than the one being retired is left alone; only one that landed back on the retiring node is retired. - The hold case speaks through the protection alone: the connection the hold is draining counts as the handoff's own, so the blanket "while quiescing" clause on the abandon path is gone and an established replacement failing during the hold is an ordinary failure, as the doc says. - The cover comments and docs say the cover stands down when every bridge's replacement is up, which is what the code does. - The drain-cap test zeroes the server latency as soon as MOVING is sent, so the replacement's handshake is not queued behind the slow replies. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 93 ++++++++----------- .../ServerEndPoint.Maintenance.cs | 20 ++-- .../MaintenanceNotificationTests.cs | 4 +- 4 files changed, 56 insertions(+), 63 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 2f9035deb..2b1ff90d6 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment the replacement is up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment every connection to that server has its replacement up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index afd473750..a13c3315c 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -35,17 +35,7 @@ internal sealed class PhysicalBridge : IDisposable /// private readonly ConcurrentQueue _backlog = new(); - /// - /// Messages that go out before anything in : a writer that lost its connection to a - /// maintenance handoff while waiting for the write lock is requeued here rather than appended, so that it - /// goes ahead of work the hold queued while it waited. This is a best effort at the ordering the writer - /// would have had, not a guarantee: a writer starved past the point where the replacement has begun - /// flushing held work lands behind what was already flushed, and nothing here orders a batch that straddles - /// the recycle - which is no more than the write path promised before. Every backlog operation drains this first. - /// - private readonly ConcurrentQueue _backlogFront = new(); - - private bool BacklogHasItems => !_backlogFront.IsEmpty || !_backlog.IsEmpty; + private bool BacklogHasItems => !_backlog.IsEmpty; private int _backlogProcessorIsRunning = 0; private int _backlogCurrentEnqueued = 0; private long _backlogTotalEnqueued = 0; @@ -173,14 +163,15 @@ internal bool HandoffPending } /// - /// Whether a connection failure belongs to a maintenance handoff in progress: the connection it retired, or an + /// Whether a connection failure belongs to a maintenance handoff in progress: the connection the hold is + /// draining or has retired (whose death, natural or not, must not abandon the work diverted off it), or an /// attempt to bring up the replacement that never completed its handshake - whatever the failure type, since /// a reset mid-handshake reports as a socket failure like any other. A replacement that was fully established /// and then failed is an ordinary failure, and a FailFast policy applies to it as usual, even while the /// handoff's protection is still up. /// private bool IsHandoffOwnFailure(PhysicalConnection connection) - => HandoffPending && (connection.RetiredForHandoff || !connection.EverFullyEstablished); + => HandoffPending && (connection.RetiredForHandoff || ReferenceEquals(connection, _quiesceTarget) || !connection.EverFullyEstablished); /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has @@ -265,16 +256,22 @@ private static bool ShouldHoldBatch(List messages) /// When the connection the hold began on has gone away by itself, replace /// whatever is current instead of declaring there is nothing to do: with no successor to pin the reconnect to, /// the ordinary reconnect may have landed straight back on the node being retired. + /// The address of the node being retired, if known, so that a replacement which + /// landed somewhere else is left alone. /// Why; for the trace. /// Bounds the wait for the write lock (together with the mutex's own timeout, whichever /// ends first); the drain re-waits until this deadline, so it is the deadline that decides how long overall. /// null if the connection could not be retired yet - caller work in flight, or the lock not /// obtained inside the deadline - so the drain should try again; otherwise whether this call, or an earlier /// one, replaced it. - internal async ValueTask TryRetireQuiesceTargetAsync(bool force, bool retireCurrentIfTargetGone, string reason, CancellationToken lockDeadline) + internal async ValueTask TryRetireQuiesceTargetAsync(bool force, bool retireCurrentIfTargetGone, System.Net.IPAddress? retiringAddress, string reason, CancellationToken lockDeadline) { var target = _quiesceTarget; - if (target is null) return false; // nothing was connected when the hold began + if (target is null) + { + ClearHandoffPending(); // nothing was connected when the hold began: nothing to retire, nothing to protect + return false; + } if (_quiesceRecycled) return true; bool retire = false; @@ -305,8 +302,16 @@ private static bool ShouldHoldBatch(List messages) _quiesceRecycled = true; return false; } - // it went away by itself and the ordinary reconnect, with no successor to pin to, may well have - // landed back on the node being retired: what is current now is what the handoff replaces + // it went away by itself and the ordinary reconnect, with no successor to pin to, may have landed + // back on the node being retired - in which case what is current now is what the handoff replaces. + // If it demonstrably landed somewhere else (DNS had already moved), it is the replacement, and + // retiring it would only force a second connect cycle. + var landedOn = (current.VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint)?.Address; + if (retiringAddress is not null && landedOn is not null && !landedOn.Equals(retiringAddress)) + { + _quiesceRecycled = true; + return false; + } target = current; _quiesceTarget = current; } @@ -572,7 +577,7 @@ public override string ToString() => MessagesSinceLastHeartbeat = (int)(Volatile.Read(ref operationCount) - Volatile.Read(ref profileLastLog)), ConnectedAt = ConnectedAt, IsWriterActive = !_singleWriter.IsAvailable, - BacklogMessagesPending = _backlogFront.Count + _backlog.Count, + BacklogMessagesPending = _backlog.Count, BacklogMessagesPendingCounter = Volatile.Read(ref _backlogCurrentEnqueued), BacklogStatus = _backlogStatus, TotalBacklogMessagesQueued = _backlogTotalEnqueued, @@ -704,7 +709,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur // is a replacement we chose, and not while a handoff is holding work back: that backlog is work diverted // off a live connection, which would have been written and answered, so a death of the old connection // during the hold (the window expiring, a blip) must leave it for the replacement to flush - if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !_quiescing + if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !IsHandoffOwnFailure(connection)) { AbandonPendingBacklog(innerException, connection); @@ -778,7 +783,7 @@ private void AbandonPendingBacklog(Exception ex, PhysicalConnection? connection, Message? next; if (locked) { - lock (_backlog) // only locked parties change the head: the timeout sweep peeks, then dequeues and compares under this lock + lock (_backlog) // the timeout sweep peeks, then dequeues and compares under this lock; a concurrent dequeue here would trip it { if (!BacklogTryDequeue(out next)) break; } @@ -1272,7 +1277,7 @@ private void BacklogEnqueue(Message message) [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool BacklogTryDequeue([NotNullWhen(true)] out Message? message) { - if (_backlogFront.TryDequeue(out message) || _backlog.TryDequeue(out message)) + if (_backlog.TryDequeue(out message)) { Interlocked.Decrement(ref _backlogCurrentEnqueued); return true; @@ -1280,16 +1285,9 @@ private bool BacklogTryDequeue([NotNullWhen(true)] out Message? message) return false; } - private bool BacklogTryPeek([NotNullWhen(true)] out Message? message) - => _backlogFront.TryPeek(out message) || _backlog.TryPeek(out message); - - /// Whether either queue holds a caller's message; snapshot enumeration, which a concurrent queue allows. + /// Whether the backlog holds a caller's message; snapshot enumeration, which a concurrent queue allows. private bool AnyCallerFacingQueued() { - foreach (var message in _backlogFront) - { - if (message.IsCallerFacing) return true; - } foreach (var message in _backlog) { if (message.IsCallerFacing) return true; @@ -1298,25 +1296,17 @@ private bool AnyCallerFacingQueued() } /// - /// Requeues, ahead of what is still queued, a caller's message that lost its connection to a maintenance - /// handoff while waiting for the write lock: it has not been written, and the work the hold queued while it - /// waited has not gone out either, so it goes first. See and the note on - /// about what this does and does not order. + /// Requeues a caller's message that lost its connection to a maintenance handoff while waiting for the write + /// lock: it has not been written, so it goes to the backlog for the replacement like the work the hold queued. + /// It is appended, not inserted at the head: a head insert would buy only a best effort at the order the + /// writer would have had (a writer starved past the start of the flush lands behind flushed work anyway), at + /// the price of a second queue and a head-mutation discipline across every backlog operation. The write path + /// never promised more than this in the same races. See . /// - private void BacklogRequeueFront(Message message) + private void RequeueForHandoff(Message message) { - bool wasEmpty = !BacklogHasItems; - Interlocked.Increment(ref _backlogCurrentEnqueued); - Interlocked.Increment(ref _backlogTotalEnqueued); - // Under the dequeuers' lock: CheckBacklogForTimeouts peeks, then dequeues and compares under this lock, - // and an insert at the head between the two would hand it a different message - which it treats as a - // threading bug, and the dequeued message would be dropped uncompleted. Only locked parties change the head. - lock (_backlog) - { - _backlogFront.Enqueue(message); - } - message.SetBacklogged(); - if (wasEmpty) Multiplexer.Root(); // as BacklogEnqueue: a backlog with a caller's task in it must pin the multiplexer + message.SetEnqueued(null); + BacklogEnqueue(message); Interlocked.MemoryBarrier(); // paired with EndQuiesce: one side or the other sees that the hold is off and the queue is not empty if (!_quiescing) StartBacklogProcessor(); } @@ -1401,7 +1391,7 @@ private void CheckBacklogForTimeouts() // Because peeking at the backlog, checking message and then dequeuing, is not thread-safe, we do have to use // a lock here, for mutual exclusion of backlog DEQUEUERS. Unfortunately. // But we reduce contention by only locking if we see something that looks timed out. - while (BacklogTryPeek(out Message? message)) + while (_backlog.TryPeek(out Message? message)) { // See if the message has pass our async timeout threshold // Note: All timed out messages must be dequeued, even when no completion is needed, to be able to dequeue and complete other timed out messages. @@ -1410,7 +1400,7 @@ private void CheckBacklogForTimeouts() { // Peek again since we didn't have lock before... // and rerun the exact same checks as above, note that it may be a different message now - if (!BacklogTryPeek(out message)) break; + if (!_backlog.TryPeek(out message)) break; if (!message.HasTimedOut(now, timeout, out var _)) break; if (!BacklogTryDequeue(out var message2) || (message != message2)) // consume it for real @@ -1693,9 +1683,8 @@ private WriteResult TimedOutBeforeWrite(Message message) /// /// For a writer that has just taken the write lock: if the connection it was headed for was retired by a /// maintenance handoff while it waited, nothing is written to it - its teardown follows the lock release. A - /// caller's message is queued for the replacement at the head of the backlog, since it was issued before - /// anything the hold has queued since; internal traffic and a message bypassing the backlog are failed as - /// having no connection, as they would be on a disconnected bridge. The connection itself says whether a + /// caller's message is queued for the replacement with the rest of the held work; internal traffic and a + /// message bypassing the backlog are failed as having no connection, as they would be on a disconnected bridge. The connection itself says whether a /// handoff retired it (), so a writer starved across any /// number of later handoffs still recognises its own. Only the handoff case: a write that loses its connection /// to an ordinary closure is still reported as a closure (#3167). @@ -1714,7 +1703,7 @@ private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, result = FailDueToNoConnection(message); return true; } - BacklogRequeueFront(message); + RequeueForHandoff(message); result = WriteResult.Success; // queued counts as success return true; } diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 022152e1d..073b6e6a6 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -484,7 +484,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? switch (decision.Action) { case HandoffAction.Recycle: - await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: false).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: false, currentAddress).ForAwait(); replaced = true; break; case HandoffAction.RecycleAtHalfWindow: @@ -492,7 +492,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // go on. Half of the *announced* window, less whatever the jitter already spent. var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); - await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successorPinned: false).ForAwait(); + await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successorPinned: false, currentAddress).ForAwait(); replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: @@ -501,7 +501,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining); - await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: true).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: true, currentAddress).ForAwait(); replaced = true; break; default: @@ -595,7 +595,9 @@ private TimeSpan DrainCap(TimeSpan budget) /// Whether reconnection is pinned to a named successor. If not, a connection that /// went away by itself during the hold may have been replaced by one on the very node being retired, and that /// replacement is retired instead; see . - private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool successorPinned) + /// The address of the node being retired, as seen from the connection that was told, + /// so that a replacement which landed elsewhere is recognised as such. + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool successorPinned, IPAddress? retiringAddress) { var watch = ValueStopwatch.StartNew(); // With less than the floor left of the window there is no hold worth having: the server is about to close @@ -605,7 +607,7 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su // The held work, and any blocking command being waited for, must not be timed out by the hold itself, nor // by the reconnect that follows it; the relaxed window covers it where relaxation is on, and the cover below - // covers it regardless, until the interactive connection is established again or this span runs out. The + // covers it regardless, until every bridge's replacement is established or this span runs out. The // reconnect can take until the end of the window plus one connect attempt: a MoveTo pins the socket target // to the named successor for the rest of the window, and a successor that does not answer is tried until // that expires before the endpoint is resolved the ordinary way. @@ -627,8 +629,8 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the // server closes it. Each bridge is released the moment its own connection is settled. - var interactiveTask = RetireBridgeAsync(interactiveBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned); - var subscriptionTask = RetireBridgeAsync(subscriptionBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned); + var interactiveTask = RetireBridgeAsync(interactiveBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned, retiringAddress); + var subscriptionTask = RetireBridgeAsync(subscriptionBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned, retiringAddress); var interactiveResult = await interactiveTask.ForAwait(); var subscriptionResult = await subscriptionTask.ForAwait(); @@ -656,7 +658,7 @@ private readonly struct BridgeRetirement(bool? recycled, bool forced) /// Retires one bridge's connection once drained, or at the cap regardless, and releases its hold; see /// . /// - private async Task RetireBridgeAsync(PhysicalBridge? bridge, ValueStopwatch watch, TimeSpan cap, string reason, bool retireCurrentIfTargetGone) + private async Task RetireBridgeAsync(PhysicalBridge? bridge, ValueStopwatch watch, TimeSpan cap, string reason, bool retireCurrentIfTargetGone, IPAddress? retiringAddress) { if (bridge is null) return new(false, false); bool forced = false; @@ -667,7 +669,7 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V { var force = watch.ElapsedMilliseconds >= cap.TotalMilliseconds; forced |= force; - var result = await bridge.TryRetireQuiesceTargetAsync(force, retireCurrentIfTargetGone, reason, lockDeadline.Token).ForAwait(); + var result = await bridge.TryRetireQuiesceTargetAsync(force, retireCurrentIfTargetGone, retiringAddress, reason, lockDeadline.Token).ForAwait(); if (result is { } done) { // whether we replaced the connection or it went away by itself, a replacement is now on its way, diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 788c5b811..6a1dca88c 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -984,6 +984,9 @@ public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); var watch = System.Diagnostics.Stopwatch.StartNew(); server.SendMoving(null, timeSeconds: 15, newEndpoint: successor, sequenceId: 0); + // the commands already accepted keep their delay; everything after this - the replacement's handshake + // above all, which the serve-one-at-a-time server would otherwise queue behind them - is answered at once + server.SetLatency(TimeSpan.Zero); var moving = await events.NextAsync(); Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); @@ -991,7 +994,6 @@ public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 12_000), "the handoff should have replaced the connection"); var recycledAfter = watch.Elapsed; - server.SetLatency(TimeSpan.Zero); // so the replacement's handshake is not slowed too log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with 12s commands in flight"); // jitter (at most a tenth of the window, capped at 1s) + the 5s cap + the lock grace (1s) + poll granularity, From cef2b906bb26810c72ee9b4f2daddb5bb5137b58 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 11:51:10 +0000 Subject: [PATCH 11/27] Handoff drain: guarded landed-on read, null-at-begin through the same path, give-up keeps the shield for diverted work Tenth round of review follow-ups: - The landed-on address is read inside a try/catch: a socket still connecting throws on RemoteEndPoint, and a concurrent teardown throws too; either counts as unknown, and unknown is retired. Previously the throw escaped, dropped the shield and abandoned the handoff in the very race the branch exists for. - A hold that began with no connection (the old one dropped during jitter or the DNS poll) now goes through the same target-gone logic on the no-successor paths, so a reconnect that landed back on the retiring node is retired instead of surviving to the window-end close. With no current connection at all there is nothing to retire or protect, and the protection lowers. - Giving up on a bridge (the lock not obtained, or an exception) stops holding new work but keeps the protection for the work already diverted: it was queued off a live connection whatever the policy, on the promise of a replacement, and the server's close plus the reconnect after it are that replacement arriving late. The protection lowers on establishment or expiry, as always. - The bridge-disposed path lowers the protection: no replacement is coming. - The fence-then-wake idiom lives in one method opposite EndQuiesce. - The drain-cap test uses one blocking command, and zeroes the server latency only once the recycle is observed: the server reads its latency when it executes a request, and zeroing it earlier raced the command the test needs to stay slow. --- src/StackExchange.Redis/PhysicalBridge.cs | 102 +++++++++++------- .../ServerEndPoint.Maintenance.cs | 12 ++- .../MaintenanceNotificationTests.cs | 21 ++-- 3 files changed, 81 insertions(+), 54 deletions(-) diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index a13c3315c..3df336ee7 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -200,10 +200,8 @@ internal void EndQuiesce() if (!_quiescing) return; _quiescing = false; _quiesceTarget = null; - // Store the flag, fence, then load the queue; an enqueuer stores the queue, fences, then loads the flag. - // With full fences on both sides at least one of them sees the other's store, so a message that lands - // as the hold is released is started by one side or the other, without spawning a processor for an - // empty backlog. StartBacklogProcessor wakes a processor that is already running. + // Store the flag, fence, then load the queue - the mirror image of EnqueueHeldAndWakeIfReleased, which is + // the only enqueue-side half of this pairing. StartBacklogProcessor wakes a processor that is already running. Interlocked.MemoryBarrier(); if (BacklogHasItems) StartBacklogProcessor(); } @@ -267,14 +265,16 @@ private static bool ShouldHoldBatch(List messages) internal async ValueTask TryRetireQuiesceTargetAsync(bool force, bool retireCurrentIfTargetGone, System.Net.IPAddress? retiringAddress, string reason, CancellationToken lockDeadline) { var target = _quiesceTarget; - if (target is null) + if (target is null && !retireCurrentIfTargetGone) { - ClearHandoffPending(); // nothing was connected when the hold began: nothing to retire, nothing to protect + // nothing was connected when the hold began, and the reconnect (pinned to the named successor) owns + // what happens next: nothing to retire, nothing to protect + ClearHandoffPending(); return false; } if (_quiesceRecycled) return true; - bool retire = false; + PhysicalConnection toRetire; bool gotLock; try { @@ -286,43 +286,61 @@ private static bool ShouldHoldBatch(List messages) } catch (ObjectDisposedException) { - return false; // the bridge was disposed mid-handoff; there is nothing left to retire + // the bridge was disposed mid-handoff: nothing left to retire, and no replacement is coming that the + // protection could be waiting for + ClearHandoffPending(); + return false; } if (!gotLock) return null; try { if (_quiesceRecycled) return true; var current = physical; - if (!ReferenceEquals(current, target)) + if (target is not null && ReferenceEquals(current, target)) { + toRetire = target; + } + else + { + // The connection the hold began on is gone - it went away by itself (the server closed it, or it + // faulted), or there was none when the hold began because it had already dropped. if (!retireCurrentIfTargetGone || current is null) { - // it went away by itself (the server closed it, or it faulted) and the reconnect, pinned to - // the named successor, owns what happens next; there is nothing left for the handoff to retire + // the reconnect, pinned to the named successor, owns what happens next; there is nothing left + // for the handoff to retire, and with no connection at all nothing to protect either _quiesceRecycled = true; + if (current is null) ClearHandoffPending(); return false; } - // it went away by itself and the ordinary reconnect, with no successor to pin to, may have landed - // back on the node being retired - in which case what is current now is what the handoff replaces. - // If it demonstrably landed somewhere else (DNS had already moved), it is the replacement, and - // retiring it would only force a second connect cycle. - var landedOn = (current.VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint)?.Address; + // With no successor to pin to, the ordinary reconnect may have landed back on the node being + // retired - in which case what is current now is what the handoff replaces. If it demonstrably + // landed somewhere else (DNS had already moved), it is the replacement, and retiring it would only + // force a second connect cycle. An address that cannot be read yet (the socket is still connecting, + // or is being torn down concurrently) counts as unknown, and unknown is retired. + System.Net.IPAddress? landedOn = null; + try + { + landedOn = (current.VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint)?.Address; + } + catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) + { + // not connected yet, or already gone: unknown + } if (retiringAddress is not null && landedOn is not null && !landedOn.Equals(retiringAddress)) { _quiesceRecycled = true; return false; } - target = current; + toRetire = current; _quiesceTarget = current; } - if (!force && target.HasCallerMessagesAwaitingResponse()) return null; + if (!force && toRetire.HasCallerMessagesAwaitingResponse()) return null; - // Under the lock only the marking: from here no writer that takes the lock writes to the target (see + // Under the lock only the marking: from here no writer that takes the lock writes to the connection (see // TryRequeueForHandoff), and the backlog processor skips it too, so the teardown itself can happen // after the release rather than stalling every writer behind event handlers and socket shutdown. - target.MarkRetiredForHandoff(); + toRetire.MarkRetiredForHandoff(); _quiesceRecycled = true; - retire = true; } finally { @@ -336,17 +354,14 @@ private static bool ShouldHoldBatch(List messages) } } - if (retire) - { - Trace($"recycling {ConnectionType} connection: {reason}"); + Trace($"recycling {ConnectionType} connection: {reason}"); - // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises - // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and - // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first - // also performs the disconnect, so the Dispose below is cleanup. - target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); - target.Dispose(); - } + // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises + // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and + // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first + // also performs the disconnect, so the Dispose below is cleanup. + toRetire.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + toRetire.Dispose(); return true; } @@ -404,16 +419,14 @@ private WriteResult QueueOrFailMessage(Message message, bool holdForHandoff = fa // Let's just never ever queue a QUIT message if (message.Command != RedisCommand.QUIT) { - message.SetEnqueued(null); - BacklogEnqueue(message); - // Note: we don't start a worker on each message here - except for held work that lands just after - // the hold was released, which nothing else would pick up until the heartbeat; see EndQuiesce for - // the fence pairing if (holdForHandoff) { - Interlocked.MemoryBarrier(); - if (!_quiescing) StartBacklogProcessor(); + EnqueueHeldAndWakeIfReleased(message); + return WriteResult.Success; } + message.SetEnqueued(null); + BacklogEnqueue(message); + // Note: we don't start a worker on each message here return WriteResult.Success; // Successfully queued, so indicate success } } @@ -1303,11 +1316,20 @@ private bool AnyCallerFacingQueued() /// the price of a second queue and a head-mutation discipline across every backlog operation. The write path /// never promised more than this in the same races. See . /// - private void RequeueForHandoff(Message message) + private void RequeueForHandoff(Message message) => EnqueueHeldAndWakeIfReleased(message); + + /// + /// Queues work held for a maintenance handoff, and makes sure it is not stranded if the hold is released + /// concurrently: the queue is stored, then a full fence, then the flag is loaded - the mirror image of + /// , which stores the flag, fences, then loads the queue. With a full fence on both + /// sides at least one of them sees the other's store, so one side or the other starts the processor; neither + /// starts one for an empty backlog. This is the only place the enqueue side of that pairing lives. + /// + private void EnqueueHeldAndWakeIfReleased(Message message) { message.SetEnqueued(null); BacklogEnqueue(message); - Interlocked.MemoryBarrier(); // paired with EndQuiesce: one side or the other sees that the hold is off and the queue is not empty + Interlocked.MemoryBarrier(); if (!_quiescing) StartBacklogProcessor(); } diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 073b6e6a6..0d9aa981a 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -678,11 +678,13 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V } if (lockDeadline.IsCancellationRequested) { - // the write lock could not be had even for the forced pass: something is stuck on it. Recycling + // The write lock could not be had even for the forced pass: something is stuck on it. Recycling // without the lock would reopen the mid-write loss the lock exists to prevent, so the connection - // is left to the server's close, which is where things stood before handoffs existed - and the - // handoff has given up on this bridge, so its pending marker goes too - bridge.ClearHandoffPending(); + // is left to the server's close, which is where things stood before handoffs existed. The hold on + // *new* work ends (the finally below); the protection stays up for the work already diverted, + // which was queued off a live connection whatever the policy on the promise of a replacement - + // the server's close and the reconnect after it are that replacement arriving late, and the + // protection lowers when it establishes or its span runs out, as always. return new(null, forced); } await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); @@ -690,8 +692,8 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V } catch (Exception ex) { + // as the give-up above: stop holding new work, keep protecting what was already diverted Multiplexer.OnInternalError(ex, EndPoint); - bridge.ClearHandoffPending(); return new(null, forced); } finally diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 6a1dca88c..9fb9994bc 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -977,16 +977,15 @@ public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() { server.SetLatency(TimeSpan.FromSeconds(12)); // well past the 5s cap and the slack below; the relaxed window covers it until the recycle var db = conn.GetDatabase(); - var blocking = Enumerable.Range(0, 3).Select(_ => db.PingAsync()).ToArray(); - Assert.True(await Poll.UntilAsync(() => conn.GetCounters().Interactive.SentItemsAwaitingResponse > 0), "the blocking commands should be in flight before MOVING arrives"); + // one, not several: the test server answers one request at a time, so each extra 12s command would hold the + // replacement's handshake behind it as well + var blocking = new[] { db.PingAsync() }; + Assert.True(await Poll.UntilAsync(() => conn.GetCounters().Interactive.SentItemsAwaitingResponse > 0), "the blocking command should be in flight before MOVING arrives"); var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); var watch = System.Diagnostics.Stopwatch.StartNew(); server.SendMoving(null, timeSeconds: 15, newEndpoint: successor, sequenceId: 0); - // the commands already accepted keep their delay; everything after this - the replacement's handshake - // above all, which the serve-one-at-a-time server would otherwise queue behind them - is answered at once - server.SetLatency(TimeSpan.Zero); var moving = await events.NextAsync(); Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); @@ -994,13 +993,17 @@ public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 12_000), "the handoff should have replaced the connection"); var recycledAfter = watch.Elapsed; - log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with 12s commands in flight"); + // Only now: the server reads its latency when it *executes* a request, and zeroing it any earlier races the + // one command this test needs to stay slow (it was observed answered at once, and then nothing was in + // flight to cap on). From here the replacement's handshake waits behind at most that one 12s reply. + server.SetLatency(TimeSpan.Zero); + log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with a 12s command in flight"); // jitter (at most a tenth of the window, capped at 1s) + the 5s cap + the lock grace (1s) + poll granularity, - // with a further second of slack: still well short of the 12s the commands need - Assert.True(recycledAfter < TimeSpan.FromSeconds(10), $"recycled after {recycledAfter}: the drain waited for the blocking commands instead of capping"); + // with a further second of slack: still well short of the 12s the command needs + Assert.True(recycledAfter < TimeSpan.FromSeconds(10), $"recycled after {recycledAfter}: the drain waited for the blocking command instead of capping"); - // the in-flight commands were failed by the recycle, which is what the server's close would have done + // the in-flight command was failed by the recycle, which is what the server's close would have done foreach (var pending in blocking) { await Assert.ThrowsAnyAsync(async () => await pending); From 95d507d1a7aee43c9e7313f71e97c5385fc500c8 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 12:23:45 +0000 Subject: [PATCH 12/27] Handoff drain: the shield keys on durable facts about the connection, not on where the handoff's flow stands Eleventh round of review follow-ups: - A connection records that it was ever the one a hold diverted work off (WasHandoffTarget). A failure is the handoff's own if the connection has that mark or never completed its handshake, whatever the handoff's flow has done since: retired by the drain, closed by the server at window end, dropped by itself, or left un-retired after a give-up. The three remaining FailFast regressions were each a flow-based clear leaking on one path; the fact-based test closes the class. - A drain pass that finds no current connection no longer lowers the protection: work may have been diverted before the target went, and the next failed reconnect must not abandon it. Only a hold that never had a target, and so diverted nothing, lowers it. - The target itself completing a handshake it was in the middle of when the hold began is not mistaken for the replacement, so the protection and the timeout cover stay up while the hold is active. - An expired protection deadline and an expired cover are cleared on observation, as GetRelaxedRemaining does, so a tick count wrapping ~24.8 days later cannot re-arm them. - The docs scope "whatever the backlogPolicy" to the drain, and say what a FailFast policy does to new commands between the replacement and its handshake. - The in-flight check is done lock-free first; the write lock is taken only for a pass that is likely to retire. The one-line requeue alias is inlined. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 75 ++++++++++++------- src/StackExchange.Redis/PhysicalConnection.cs | 10 +++ .../ServerEndPoint.Maintenance.cs | 14 +++- 4 files changed, 68 insertions(+), 33 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 2b1ff90d6..7ee167b90 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). This happens whatever the `backlogPolicy`, because the connection is live and the command would otherwise have been written; and a `FailFast` policy does not abandon that held work when the old connection is closed, since a handoff is a replacement the client chose rather than a failure. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment every connection to that server has its replacement up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment every connection to that server has its replacement up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 3df336ee7..f1025df53 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -158,20 +158,26 @@ internal bool HandoffPending get { var deadline = Volatile.Read(ref _handoffPendingDeadlineTicks); - return deadline != 0 && unchecked(deadline - Environment.TickCount) > 0; + if (deadline == 0) return false; + if (unchecked(deadline - Environment.TickCount) > 0) return true; + // expired: clear it, as GetRelaxedRemaining does, so that a tick count wrapping ~24.8 days later cannot + // bring a stale deadline back to life + Interlocked.CompareExchange(ref _handoffPendingDeadlineTicks, 0, deadline); + return false; } } /// - /// Whether a connection failure belongs to a maintenance handoff in progress: the connection the hold is - /// draining or has retired (whose death, natural or not, must not abandon the work diverted off it), or an - /// attempt to bring up the replacement that never completed its handshake - whatever the failure type, since - /// a reset mid-handshake reports as a socket failure like any other. A replacement that was fully established - /// and then failed is an ordinary failure, and a FailFast policy applies to it as usual, even while the - /// handoff's protection is still up. + /// Whether a connection failure belongs to a maintenance handoff in progress. Two durable facts about the + /// connection decide it, not where the handoff's flow happens to stand: it was ever the connection a hold + /// diverted work off (retired by the drain, closed by the server at the end of the window, or dropped by + /// itself - its death must not abandon that work), or it never completed its handshake (an attempt to bring + /// up the replacement, whatever the failure type, since a reset mid-handshake reports as a socket failure like + /// any other). A replacement that was fully established and then failed is an ordinary failure, and a FailFast + /// policy applies to it as usual, even while the handoff's protection is still up. /// private bool IsHandoffOwnFailure(PhysicalConnection connection) - => HandoffPending && (connection.RetiredForHandoff || ReferenceEquals(connection, _quiesceTarget) || !connection.EverFullyEstablished); + => HandoffPending && (connection.WasHandoffTarget || !connection.EverFullyEstablished); /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has @@ -189,7 +195,9 @@ private bool IsHandoffOwnFailure(PhysicalConnection connection) /// the timeout cover on the server - if no replacement establishes first. internal void BeginQuiesce(TimeSpan protection) { - _quiesceTarget = physical; + var target = physical; + target?.MarkHandoffTarget(); + _quiesceTarget = target; _quiesceRecycled = false; Volatile.Write(ref _handoffPendingDeadlineTicks, ServerEndPoint.NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)protection.TotalMilliseconds))); _quiescing = true; @@ -274,6 +282,11 @@ private static bool ShouldHoldBatch(List messages) } if (_quiesceRecycled) return true; + // Lock-free first: the drain polls this every 20ms, and the in-flight question only needs the connection's + // own lock. The write lock is taken only for a pass that is likely to retire, where it has to be held + // across the check and the marking. + if (!force && target is not null && ReferenceEquals(physical, target) && target.HasCallerMessagesAwaitingResponse()) return null; + PhysicalConnection toRetire; bool gotLock; try @@ -306,10 +319,12 @@ private static bool ShouldHoldBatch(List messages) // faulted), or there was none when the hold began because it had already dropped. if (!retireCurrentIfTargetGone || current is null) { - // the reconnect, pinned to the named successor, owns what happens next; there is nothing left - // for the handoff to retire, and with no connection at all nothing to protect either + // The reconnect, pinned to the named successor, owns what happens next; there is nothing left + // for the handoff to retire. The protection stays up for whatever the hold diverted off the + // target before it went: the next failed reconnect attempt must not abandon it. Only a hold + // that never had a target can have diverted nothing, and only then is there nothing to protect. _quiesceRecycled = true; - if (current is null) ClearHandoffPending(); + if (target is null) ClearHandoffPending(); return false; } // With no successor to pin to, the ordinary reconnect may have landed back on the node being @@ -332,6 +347,7 @@ private static bool ShouldHoldBatch(List messages) return false; } toRetire = current; + current.MarkHandoffTarget(); _quiesceTarget = current; } if (!force && toRetire.HasCallerMessagesAwaitingResponse()) return null; @@ -825,9 +841,12 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) { reportNextFailure = reconfigureNextFailure = true; LastException = null; - // a connection establishing after the hold began is the handoff's replacement: the protection stands - // down for this bridge (ServerEndPoint.OnFullyEstablished lowers the shared cover once no bridge needs it) - Volatile.Write(ref _handoffPendingDeadlineTicks, 0); + // a connection establishing after the hold began is the handoff's replacement - unless it is the very + // connection the hold is draining, completing a handshake it was in the middle of when the hold began, + // in which case the hold is active and nothing has been replaced yet. For a replacement the protection + // stands down for this bridge (ServerEndPoint.OnFullyEstablished lowers the shared cover once no bridge + // needs it). + if (!connection.WasHandoffTarget) Volatile.Write(ref _handoffPendingDeadlineTicks, 0); Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -1309,22 +1328,19 @@ private bool AnyCallerFacingQueued() } /// - /// Requeues a caller's message that lost its connection to a maintenance handoff while waiting for the write - /// lock: it has not been written, so it goes to the backlog for the replacement like the work the hold queued. - /// It is appended, not inserted at the head: a head insert would buy only a best effort at the order the - /// writer would have had (a writer starved past the start of the flush lands behind flushed work anyway), at - /// the price of a second queue and a head-mutation discipline across every backlog operation. The write path - /// never promised more than this in the same races. See . - /// - private void RequeueForHandoff(Message message) => EnqueueHeldAndWakeIfReleased(message); - - /// - /// Queues work held for a maintenance handoff, and makes sure it is not stranded if the hold is released + /// Queues work held for a maintenance handoff - diverted at the gate, or requeued by a writer that reached the + /// lock to find its connection retired - and makes sure it is not stranded if the hold is released /// concurrently: the queue is stored, then a full fence, then the flag is loaded - the mirror image of /// , which stores the flag, fences, then loads the queue. With a full fence on both /// sides at least one of them sees the other's store, so one side or the other starts the processor; neither /// starts one for an empty backlog. This is the only place the enqueue side of that pairing lives. /// + /// + /// A requeued message is appended, not inserted at the head: a head insert would buy only a best effort at the + /// order the writer would have had (a writer starved past the start of the flush lands behind flushed work + /// anyway), at the price of a second queue and a head-mutation discipline across every backlog operation. The + /// write path never promised more than this in the same races. + /// private void EnqueueHeldAndWakeIfReleased(Message message) { message.SetEnqueued(null); @@ -1705,8 +1721,9 @@ private WriteResult TimedOutBeforeWrite(Message message) /// /// For a writer that has just taken the write lock: if the connection it was headed for was retired by a /// maintenance handoff while it waited, nothing is written to it - its teardown follows the lock release. A - /// caller's message is queued for the replacement with the rest of the held work; internal traffic and a - /// message bypassing the backlog are failed as having no connection, as they would be on a disconnected bridge. The connection itself says whether a + /// caller's message is queued for the replacement with the rest of the held work (appended, see + /// ); internal traffic and a message bypassing the backlog are failed + /// as having no connection, as they would be on a disconnected bridge. The connection itself says whether a /// handoff retired it (), so a writer starved across any /// number of later handoffs still recognises its own. Only the handoff case: a write that loses its connection /// to an ordinary closure is still reported as a closure (#3167). @@ -1725,7 +1742,7 @@ private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, result = FailDueToNoConnection(message); return true; } - RequeueForHandoff(message); + EnqueueHeldAndWakeIfReleased(message); result = WriteResult.Success; // queued counts as success return true; } diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index e45d4b261..73a7110a8 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -986,6 +986,16 @@ internal void GetStormLog(StringBuilder sb) internal void MarkRetiredForHandoff() => RetiredForHandoff = true; + /// + /// Whether this connection was ever the one a maintenance handoff held work back from. A durable fact about the + /// connection rather than about where the handoff's flow stands: whatever happens to it afterwards - retired by + /// the drain, closed by the server at the end of the window, dropped by itself - its failure is the handoff's + /// own, and work diverted off it must not be abandoned on that account. Never reset. + /// + internal bool WasHandoffTarget { get; private set; } + + internal void MarkHandoffTarget() => WasHandoffTarget = true; + /// /// Whether this connection ever completed its handshake. A failure on a connection that never did is a failed /// attempt to bring a connection up, which is how a maintenance handoff tells "the replacement is not up yet" diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 0d9aa981a..35319b204 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -209,10 +209,18 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) // (zero disables it), those commands must not be timed out by the hold itself, so the hold imposes its // own floor on the effective timeout for as long as it, and the reconnect after it, can take var cover = Volatile.Read(ref _holdCoverDeadlineTicks); - if (cover != 0 && unchecked(cover - Environment.TickCount) > 0) + if (cover != 0) { - var floor = Volatile.Read(ref _holdCoverMilliseconds); - if (floor > result) result = floor; + if (unchecked(cover - Environment.TickCount) > 0) + { + var floor = Volatile.Read(ref _holdCoverMilliseconds); + if (floor > result) result = floor; + } + else + { + // expired: clear it, as GetRelaxedRemaining does, so a wrapped tick count cannot re-arm a stale floor + Interlocked.CompareExchange(ref _holdCoverDeadlineTicks, 0, cover); + } } return result; } From 5b10b77f94093062c6e031db19ec97561712d34e Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 13:10:21 +0000 Subject: [PATCH 13/27] Handoff drain: protection stands down only for a bona fide replacement; one tick-deadline type Twelfth round of review follow-ups: - A connection is a bona fide replacement when it is not the one the hold began on and, unless reconnection is pinned to a named successor, has demonstrably landed somewhere other than the node being retired - by address and port, since Redis Enterprise rebinds to other ports on the same host. Only such a connection lowers the protection when it establishes, ends the hold on new work and receives the held backlog. A reconnect that landed back on the retiring node (no successor, DNS still stale) is neither target nor replacement: new work is held off it, the backlog does not flush to it, and the drain retires it in its turn. - The protection deadline is monotonic: a hold beginning while an earlier handoff's protection is up extends it and remembers that older diverted work exists, so a hold with no target of its own does not clear it. - TickDeadline holds the sentinel, nudge, monotonic-extend and clear-on- expiry invariants once; the relaxed window, the hold cover and the per-bridge protection all use it. - The three handoff flags on the connection are volatile. - A requeued message keeps its write time; it only forgets the connection it was headed for. --- .../Maintenance/TickDeadline.cs | 66 +++++++ src/StackExchange.Redis/Message.cs | 11 ++ src/StackExchange.Redis/PhysicalBridge.cs | 177 ++++++++++-------- src/StackExchange.Redis/PhysicalConnection.cs | 16 +- .../ServerEndPoint.Maintenance.cs | 100 ++++------ 5 files changed, 224 insertions(+), 146 deletions(-) create mode 100644 src/StackExchange.Redis/Maintenance/TickDeadline.cs diff --git a/src/StackExchange.Redis/Maintenance/TickDeadline.cs b/src/StackExchange.Redis/Maintenance/TickDeadline.cs new file mode 100644 index 000000000..ec2256d59 --- /dev/null +++ b/src/StackExchange.Redis/Maintenance/TickDeadline.cs @@ -0,0 +1,66 @@ +using System; +using System.Threading; + +namespace StackExchange.Redis.Maintenance; + +/// +/// A deadline expressed in terms, where zero means "unset". One place for the +/// three invariants every such field needs: a genuine zero tick count is nudged to one so it cannot read as unset; +/// an observed expiry is cleared (by compare-exchange, so a concurrent extension is not undone), so that a tick count +/// wrapping ~24.8 days later cannot bring a stale deadline back to life; and extension is monotonic, never shortening +/// a deadline that is already later. Used for the relaxed-timeout window, the handoff hold cover and the per-bridge +/// handoff protection, which all had their own copies of this before. +/// +internal struct TickDeadline +{ + private int _ticks; + + /// The raw deadline, or zero when unset. For callers that record *when* a window was due to end. + public int Ticks => Volatile.Read(ref _ticks); + + public void SetIn(int milliseconds) => Volatile.Write(ref _ticks, NudgeFromZero(unchecked(Environment.TickCount + milliseconds))); + + public void Clear() => Volatile.Write(ref _ticks, 0); + + /// + /// Moves the deadline out to now plus unless it is already later than that. + /// + /// How far from now the deadline should be, at least. + /// The deadline now in force, whether or not this call set it. + /// Whether this call extended it. + public bool TryExtendBy(int milliseconds, out int deadline) + { + var candidate = NudgeFromZero(unchecked(Environment.TickCount + milliseconds)); + while (true) + { + var current = Volatile.Read(ref _ticks); + if (current != 0 && unchecked(candidate - current) <= 0) + { + deadline = current; + return false; + } + if (Interlocked.CompareExchange(ref _ticks, candidate, current) == current) + { + deadline = candidate; + return true; + } + } + } + + /// Milliseconds until the deadline, or zero if unset or passed; a passed deadline is cleared. + public int RemainingMilliseconds() + { + var deadline = Volatile.Read(ref _ticks); + if (deadline == 0) return 0; + var remaining = unchecked(deadline - Environment.TickCount); + if (remaining > 0) return remaining; + + // expired: clear it, but only if nobody has moved it on in the meantime + Interlocked.CompareExchange(ref _ticks, 0, deadline); + return 0; + } + + public bool IsActive => RemainingMilliseconds() > 0; + + private static int NudgeFromZero(int ticks) => ticks == 0 ? 1 : ticks; +} diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index e6943cbcf..055b361c9 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -894,6 +894,17 @@ internal void SetWriteTime() /// internal void InheritWriteTime(Message parent) => _writeTickCount = parent._writeTickCount; + /// + /// The message is going back to the backlog after the connection it was headed for was retired by a maintenance + /// handoff: it no longer points at that connection, but it keeps its write time, since the budget it has already + /// spent (including waiting for the write lock) still counts. Unlike , which restarts it. + /// + internal void SetRequeued() + { + _enqueuedTo = null; + _queuedStampSent = _queuedStampReceived = -1; + } + private int _writeTickCount; public int GetWriteTime() => Volatile.Read(ref _writeTickCount); diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index f1025df53..5ed167b3f 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -133,37 +133,56 @@ public enum State : byte /// internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); - private volatile bool _quiescing; + private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceInheritedProtection; private PhysicalConnection? _quiesceTarget; - private volatile bool _quiesceRecycled; + private System.Net.IPEndPoint? _quiesceRetiringEndPoint; /// - /// While set, a maintenance handoff is replacing this bridge's connection, and work the hold diverted into the - /// backlog survives the reconnect under a FailFast policy as well as the hold itself: a failure on the way to - /// the replacement is not a reason to abandon it. Raised in ; lowered when a - /// replacement is fully established, when the handoff gives up on this bridge, or by the deadline. The server's - /// timeout cover shares exactly this lifecycle, across both bridges. + /// While active, a maintenance handoff is replacing this bridge's connection, and work the hold diverted into + /// the backlog survives the reconnect under a FailFast policy as well as the hold itself: a failure on the way + /// to the replacement is not a reason to abandon it. Raised (monotonically: a later hold may extend it, never + /// shorten it, since an earlier hold's diverted work may still be queued) in ; + /// lowered when a bona fide replacement is fully established, when the handoff finds it has nothing to protect, + /// or by the deadline. The server's timeout cover shares exactly this lifecycle, across both bridges. /// - private int _handoffPendingDeadlineTicks; + private Maintenance.TickDeadline _handoffProtection; - /// The handoff has given up on this bridge: its protection stands down. + /// The handoff has nothing left to protect on this bridge: its protection stands down. internal void ClearHandoffPending() { - Volatile.Write(ref _handoffPendingDeadlineTicks, 0); + _handoffProtection.Clear(); ServerEndPoint.OnHandoffProtectionLowered(); } - internal bool HandoffPending + internal bool HandoffPending => _handoffProtection.IsActive; + + /// + /// Whether a connection is a bona fide replacement for the one a handoff is retiring: not that connection + /// itself, and - where reconnection is not pinned to a named successor, so the ordinary reconnect may have + /// landed straight back on the node being retired - demonstrably somewhere else, by address *and* port, since + /// Redis Enterprise regularly rebinds to another port on the same host. A connection whose address cannot be + /// read yet is not known to be a replacement. Only a bona fide replacement establishing lowers the protection, + /// releases the hold on new work, and receives the held backlog. + /// + private bool IsBonaFideReplacement(PhysicalConnection connection) + { + if (connection.WasHandoffTarget) return false; + if (_quiesceSuccessorPinned) return true; + var retiring = _quiesceRetiringEndPoint; + if (retiring is null) return true; // nothing known to compare against + var landedOn = RemoteEndPointOrNull(connection); + return landedOn is not null && !landedOn.Equals(retiring); + } + + private static System.Net.IPEndPoint? RemoteEndPointOrNull(PhysicalConnection connection) { - get + try { - var deadline = Volatile.Read(ref _handoffPendingDeadlineTicks); - if (deadline == 0) return false; - if (unchecked(deadline - Environment.TickCount) > 0) return true; - // expired: clear it, as GetRelaxedRemaining does, so that a tick count wrapping ~24.8 days later cannot - // bring a stale deadline back to life - Interlocked.CompareExchange(ref _handoffPendingDeadlineTicks, 0, deadline); - return false; + return connection.VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint; + } + catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) + { + return null; // not connected yet, or already gone: unknown } } @@ -192,14 +211,24 @@ private bool IsHandoffOwnFailure(PhysicalConnection connection) /// was in flight at that instant. /// /// For how long at most the handoff's protection stands - the FailFast shield here and - /// the timeout cover on the server - if no replacement establishes first. - internal void BeginQuiesce(TimeSpan protection) + /// the timeout cover on the server - if no bona fide replacement establishes first. + /// The address and port of the node being retired, as seen from the connection + /// that was told, if known; see . + /// Whether reconnection is pinned to a named successor, in which case any + /// connection other than the target is a replacement. + internal void BeginQuiesce(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, bool successorPinned) { var target = physical; target?.MarkHandoffTarget(); _quiesceTarget = target; + _quiesceRetiringEndPoint = retiringEndPoint; + _quiesceSuccessorPinned = successorPinned; _quiesceRecycled = false; - Volatile.Write(ref _handoffPendingDeadlineTicks, ServerEndPoint.NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)protection.TotalMilliseconds))); + // a hold that begins while an earlier handoff's protection is still up (the bridge still reconnecting from + // it, with that handoff's diverted work queued) extends it rather than replacing it, and remembers that + // there is older work to protect even if this hold itself diverts nothing + _quiesceInheritedProtection = _handoffProtection.IsActive; + _handoffProtection.TryExtendBy((int)protection.TotalMilliseconds, out _); _quiescing = true; } @@ -223,14 +252,17 @@ internal void EndQuiesce() private bool ShouldHoldForHandoff(Message message) => HoldActive && IsHoldable(message); /// - /// A handoff is draining the connection that is current right now. The moment the target is gone - retired - /// by the drain, or dead of its own accord with a replacement already establishing - there is nothing to hold - /// work back from, and the backlog may flush to whatever is current. + /// A handoff's hold has not yet been resolved: the connection it began on has not been retired, and whatever is + /// current is not a bona fide replacement either (it may be the target still, or a reconnect that landed back + /// on the node being retired). While unresolved, new work is held and the backlog does not flush, because + /// whatever is current is about to be retired. The moment a bona fide replacement is current, or the retire has + /// happened, the hold is over and the backlog may flush to whatever is current. /// - private bool HoldingCurrentConnection => _quiescing && _quiesceTarget is { } target && ReferenceEquals(physical, target); + private bool HoldUnresolved + => _quiescing && !_quiesceRecycled && !(physical is { } current && !ReferenceEquals(current, _quiesceTarget) && IsBonaFideReplacement(current)); - /// A handoff is draining the current connection, and it is live. - private bool HoldActive => HoldingCurrentConnection && IsConnected && !NeedsReconnect; + /// A handoff's hold is unresolved, and the current connection is live. + private bool HoldActive => HoldUnresolved && IsConnected && !NeedsReconnect; /// A caller's work, and not QUIT, which is never queued. private static bool IsHoldable(Message message) => !message.IsInternalCall && message.Command != RedisCommand.QUIT; @@ -252,34 +284,24 @@ private static bool ShouldHoldBatch(List messages) /// /// Retires the connection the hold began on, under the write lock, once nothing a caller is waiting on is in - /// flight on it - or regardless, when says the drain has waited long enough. Only - /// that connection: one that replaced it meanwhile is never touched, however often this is called. The lock - /// is what keeps a writer that passed the hold check before it was set from slipping onto the old connection - /// between the check and the recycle; such a writer finds the connection replaced when it gets the lock, and - /// queues for the replacement instead (see ). + /// flight on it - or regardless, when says the drain has waited long enough. If that + /// connection has gone by itself and what is current is a reconnect that landed back on the node being retired + /// (no successor pinned, DNS still naming the old node), that reconnect takes its place and is retired instead; + /// a bona fide replacement is left alone. The lock is what keeps a writer that passed the hold check before it + /// was set from slipping onto the old connection between the check and the retire; such a writer finds the + /// connection retired when it gets the lock, and queues for the replacement instead (see + /// ). /// /// Replace the connection even if caller work is still in flight on it. - /// When the connection the hold began on has gone away by itself, replace - /// whatever is current instead of declaring there is nothing to do: with no successor to pin the reconnect to, - /// the ordinary reconnect may have landed straight back on the node being retired. - /// The address of the node being retired, if known, so that a replacement which - /// landed somewhere else is left alone. /// Why; for the trace. /// Bounds the wait for the write lock (together with the mutex's own timeout, whichever /// ends first); the drain re-waits until this deadline, so it is the deadline that decides how long overall. /// null if the connection could not be retired yet - caller work in flight, or the lock not /// obtained inside the deadline - so the drain should try again; otherwise whether this call, or an earlier /// one, replaced it. - internal async ValueTask TryRetireQuiesceTargetAsync(bool force, bool retireCurrentIfTargetGone, System.Net.IPAddress? retiringAddress, string reason, CancellationToken lockDeadline) + internal async ValueTask TryRetireQuiesceTargetAsync(bool force, string reason, CancellationToken lockDeadline) { var target = _quiesceTarget; - if (target is null && !retireCurrentIfTargetGone) - { - // nothing was connected when the hold began, and the reconnect (pinned to the named successor) owns - // what happens next: nothing to retire, nothing to protect - ClearHandoffPending(); - return false; - } if (_quiesceRecycled) return true; // Lock-free first: the drain polls this every 20ms, and the in-flight question only needs the connection's @@ -317,35 +339,26 @@ private static bool ShouldHoldBatch(List messages) { // The connection the hold began on is gone - it went away by itself (the server closed it, or it // faulted), or there was none when the hold began because it had already dropped. - if (!retireCurrentIfTargetGone || current is null) + if (current is null) { - // The reconnect, pinned to the named successor, owns what happens next; there is nothing left - // for the handoff to retire. The protection stays up for whatever the hold diverted off the - // target before it went: the next failed reconnect attempt must not abandon it. Only a hold - // that never had a target can have diverted nothing, and only then is there nothing to protect. + // Nothing to retire. Whatever the hold diverted before the target went stays protected for the + // reconnect that is coming; only a hold that never had a target, and inherited no earlier + // handoff's protection, can have nothing to protect. _quiesceRecycled = true; - if (target is null) ClearHandoffPending(); + if (target is null && !_quiesceInheritedProtection) ClearHandoffPending(); return false; } - // With no successor to pin to, the ordinary reconnect may have landed back on the node being - // retired - in which case what is current now is what the handoff replaces. If it demonstrably - // landed somewhere else (DNS had already moved), it is the replacement, and retiring it would only - // force a second connect cycle. An address that cannot be read yet (the socket is still connecting, - // or is being torn down concurrently) counts as unknown, and unknown is retired. - System.Net.IPAddress? landedOn = null; - try - { - landedOn = (current.VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint)?.Address; - } - catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) - { - // not connected yet, or already gone: unknown - } - if (retiringAddress is not null && landedOn is not null && !landedOn.Equals(retiringAddress)) + if (IsBonaFideReplacement(current)) { + // the reconnect went somewhere else (a pinned successor, or DNS had already moved): it is the + // replacement, and retiring it would only force a second connect cycle. If it is already up, the + // protection stands down now; if it is still connecting, its establishment will do that. _quiesceRecycled = true; + if (current.EverFullyEstablished) ClearHandoffPending(); return false; } + // back on the node being retired (or not yet known to be anywhere else): what is current now is + // what the handoff replaces toRetire = current; current.MarkHandoffTarget(); _quiesceTarget = current; @@ -841,12 +854,11 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) { reportNextFailure = reconfigureNextFailure = true; LastException = null; - // a connection establishing after the hold began is the handoff's replacement - unless it is the very - // connection the hold is draining, completing a handshake it was in the middle of when the hold began, - // in which case the hold is active and nothing has been replaced yet. For a replacement the protection - // stands down for this bridge (ServerEndPoint.OnFullyEstablished lowers the shared cover once no bridge - // needs it). - if (!connection.WasHandoffTarget) Volatile.Write(ref _handoffPendingDeadlineTicks, 0); + // only a bona fide replacement establishing lowers the protection for this bridge (and then + // ServerEndPoint.OnFullyEstablished lowers the shared cover once no bridge needs it): not the very + // connection the hold is draining completing a handshake it was in the middle of, and not a reconnect + // that landed back on the node being retired, which the drain is about to retire in its turn + if (IsBonaFideReplacement(connection)) _handoffProtection.Clear(); Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -1341,9 +1353,13 @@ private bool AnyCallerFacingQueued() /// anyway), at the price of a second queue and a head-mutation discipline across every backlog operation. The /// write path never promised more than this in the same races. /// - private void EnqueueHeldAndWakeIfReleased(Message message) + /// The message to queue. + /// The message had already been enqueued (to the connection that has since been retired): + /// it keeps its write time, since the budget it has spent still counts, and only forgets that connection. + private void EnqueueHeldAndWakeIfReleased(Message message, bool requeue = false) { - message.SetEnqueued(null); + if (requeue) message.SetRequeued(); + else message.SetEnqueued(null); BacklogEnqueue(message); Interlocked.MemoryBarrier(); if (!_quiescing) StartBacklogProcessor(); @@ -1592,9 +1608,10 @@ private void ProcessBridgeBacklog() // check whether the backlog is empty *before* even trying to get the lock if (!BacklogHasItems) return; // nothing to do - // a maintenance handoff is holding work back from this connection: nothing would be written, and - // the lock is better left to the drain and the writers it is waiting on; the release kicks this again - if (HoldingCurrentConnection) return; + // a maintenance handoff is holding work back from the connection about to be retired: nothing would + // be written, and the lock is better left to the drain and the writers it is waiting on; the + // release kicks this again + if (HoldUnresolved) return; // try and get the lock; if unsuccessful, retry gotLock = _singleWriter.TryTakeSync(); @@ -1606,7 +1623,7 @@ private void ProcessBridgeBacklog() // Timeouts are handled above, so we're exclusively into backlog items eligible to write at this point. // If we can't write them, abort and wait for the next heartbeat or activation to try this again. bool flush = false; - while (IsConnected && !HoldingCurrentConnection) // not while a handoff is holding work back from this connection + while (IsConnected && !HoldUnresolved) // not while a handoff is holding work back from the connection about to be retired { // Snapshot the connection for the whole of this message: OnDisconnected nulls the field // without waiting for the write lock, so re-reading it (as the loop guard used to) can hand @@ -1742,7 +1759,7 @@ private bool TryRequeueForHandoff(PhysicalConnection physical, Message message, result = FailDueToNoConnection(message); return true; } - EnqueueHeldAndWakeIfReleased(message); + EnqueueHeldAndWakeIfReleased(message, requeue: true); result = WriteResult.Success; // queued counts as success return true; } diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 73a7110a8..eea126ffb 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -982,9 +982,13 @@ internal void GetStormLog(StringBuilder sb) /// reached the write lock afterwards can tell a planned replacement from an ordinary closure. Set under the /// bridge's write lock; the report and dispose follow once the lock is released. Never reset. /// - internal bool RetiredForHandoff { get; private set; } + internal bool RetiredForHandoff => _retiredForHandoff; - internal void MarkRetiredForHandoff() => RetiredForHandoff = true; + internal void MarkRetiredForHandoff() => _retiredForHandoff = true; + + // volatile: each is written on one thread and read on others as part of the handoff's identity test, in the + // very races that test exists for; a plain field could be read stale exactly then + private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished; /// /// Whether this connection was ever the one a maintenance handoff held work back from. A durable fact about the @@ -992,18 +996,18 @@ internal void GetStormLog(StringBuilder sb) /// the drain, closed by the server at the end of the window, dropped by itself - its failure is the handoff's /// own, and work diverted off it must not be abandoned on that account. Never reset. /// - internal bool WasHandoffTarget { get; private set; } + internal bool WasHandoffTarget => _wasHandoffTarget; - internal void MarkHandoffTarget() => WasHandoffTarget = true; + internal void MarkHandoffTarget() => _wasHandoffTarget = true; /// /// Whether this connection ever completed its handshake. A failure on a connection that never did is a failed /// attempt to bring a connection up, which is how a maintenance handoff tells "the replacement is not up yet" /// from "the replacement was up and then failed" without enumerating failure types. /// - internal bool EverFullyEstablished { get; private set; } + internal bool EverFullyEstablished => _everFullyEstablished; - internal void MarkFullyEstablished() => EverFullyEstablished = true; + internal void MarkFullyEstablished() => _everFullyEstablished = true; /// /// Whether any message awaiting a response was issued by a *caller* rather than by us. diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 35319b204..5bad3a6a7 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -159,7 +159,7 @@ private void ReconcileMaintenanceNotifications(PhysicalConnection connection) // can be read without a lock from the heartbeat sweeps. Zero means "no window"; a deadline that computes // to zero is nudged by a tick rather than complicating the sentinel. Comparisons are unchecked // subtraction, as elsewhere in this type, so the ~49-day wrap is a non-event. - private int _relaxedDeadlineTicks; + private TickDeadline _relaxedDeadline; // the notification that last touched the window, reported on faults that happen inside it private int _relaxedType; @@ -208,24 +208,16 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) // a handoff hold diverts commands off a live connection into the backlog; whatever the relaxation setting // (zero disables it), those commands must not be timed out by the hold itself, so the hold imposes its // own floor on the effective timeout for as long as it, and the reconnect after it, can take - var cover = Volatile.Read(ref _holdCoverDeadlineTicks); - if (cover != 0) + if (_holdCover.IsActive) { - if (unchecked(cover - Environment.TickCount) > 0) - { - var floor = Volatile.Read(ref _holdCoverMilliseconds); - if (floor > result) result = floor; - } - else - { - // expired: clear it, as GetRelaxedRemaining does, so a wrapped tick count cannot re-arm a stale floor - Interlocked.CompareExchange(ref _holdCoverDeadlineTicks, 0, cover); - } + var floor = Volatile.Read(ref _holdCoverMilliseconds); + if (floor > result) result = floor; } return result; } - private int _holdCoverDeadlineTicks, _holdCoverMilliseconds; + private TickDeadline _holdCover; + private int _holdCoverMilliseconds; /// /// For the duration of a handoff hold plus the reconnect that follows it - including the rest of the window, for @@ -238,7 +230,7 @@ private void SetHoldCover(TimeSpan span) { var ms = (int)span.TotalMilliseconds; Volatile.Write(ref _holdCoverMilliseconds, ms); - Volatile.Write(ref _holdCoverDeadlineTicks, NudgeFromZeroTicks(unchecked(Environment.TickCount + ms))); + _holdCover.SetIn(ms); } /// @@ -246,7 +238,7 @@ private void SetHoldCover(TimeSpan span) /// ordinary timeouts - including a deliberately short one, with relaxation turned off - apply once more. The /// deadline set by is only the bound for the case where that never happens. /// - private void ClearHoldCover() => Volatile.Write(ref _holdCoverDeadlineTicks, 0); + private void ClearHoldCover() => _holdCover.Clear(); /// Whether either bridge still has a handoff's protection up; see . internal bool AnyHandoffPending => interactive?.HandoffPending == true || subscription?.HandoffPending == true; @@ -263,18 +255,7 @@ internal void OnHandoffProtectionLowered() /// /// Milliseconds of relaxation left, or zero if none; clears an expired window as a side-effect. /// - private int GetRelaxedRemaining() - { - var deadline = Volatile.Read(ref _relaxedDeadlineTicks); - if (deadline == 0) return 0; - - var remaining = unchecked(deadline - Environment.TickCount); - if (remaining > 0) return remaining; - - // expired; clear it, but only if nobody has moved it on in the meantime - Interlocked.CompareExchange(ref _relaxedDeadlineTicks, 0, deadline); - return 0; - } + private int GetRelaxedRemaining() => _relaxedDeadline.RemainingMilliseconds(); /// /// An announced disruption has started (or is still running): open or extend the relaxed window. @@ -340,15 +321,14 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; if (tail <= TimeSpan.Zero) { - Volatile.Write(ref _relaxedDeadlineTicks, 0); + _relaxedDeadline.Clear(); Volatile.Write(ref _relaxedEndedTicks, NudgeFromZeroTicks(Environment.TickCount)); Multiplexer.Trace($"{type}: relaxation ended", ToString()); return; } - var deadline = NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)tail.TotalMilliseconds)); - Volatile.Write(ref _relaxedDeadlineTicks, deadline); - Volatile.Write(ref _relaxedEndedTicks, deadline); + _relaxedDeadline.SetIn((int)tail.TotalMilliseconds); + Volatile.Write(ref _relaxedEndedTicks, _relaxedDeadline.Ticks); Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } @@ -452,12 +432,20 @@ internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalC var budget = window is { } value && value > TimeSpan.Zero ? value : Multiplexer.RawConfig.MaintenanceRelaxedTimeout; // no window given: use the relaxation floor - var current = (connection.VolatileSocket?.RemoteEndPoint as IPEndPoint)?.Address; + IPEndPoint? current = null; + try + { + current = connection.VolatileSocket?.RemoteEndPoint as IPEndPoint; + } + catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) + { + // the socket is gone or not connected: no address to compare against + } _ = HandoffAsync(budget, successor, current); } - private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? currentAddress) + private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint? currentEndPoint) { var watch = ValueStopwatch.StartNew(); var replaced = false; @@ -472,7 +460,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? var decision = await MaintenanceHandoff.DecideAsync( EndPoint, successor, - currentAddress, + currentEndPoint?.Address, remaining, pollInterval: TimeSpan.FromSeconds(1), // records carry a 5s TTL, so this is several looks per record resolve: Multiplexer.AddressResolver, @@ -492,7 +480,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? switch (decision.Action) { case HandoffAction.Recycle: - await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: false, currentAddress).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: false, currentEndPoint).ForAwait(); replaced = true; break; case HandoffAction.RecycleAtHalfWindow: @@ -500,7 +488,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // go on. Half of the *announced* window, less whatever the jitter already spent. var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); - await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successorPinned: false, currentAddress).ForAwait(); + await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successorPinned: false, currentEndPoint).ForAwait(); replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: @@ -509,7 +497,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPAddress? // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining); - await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: true, currentAddress).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: true, currentEndPoint).ForAwait(); replaced = true; break; default: @@ -603,9 +591,9 @@ private TimeSpan DrainCap(TimeSpan budget) /// Whether reconnection is pinned to a named successor. If not, a connection that /// went away by itself during the hold may have been replaced by one on the very node being retired, and that /// replacement is retired instead; see . - /// The address of the node being retired, as seen from the connection that was told, - /// so that a replacement which landed elsewhere is recognised as such. - private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool successorPinned, IPAddress? retiringAddress) + /// The address and port of the node being retired, as seen from the connection that + /// was told, so that a replacement which landed elsewhere is recognised as such. + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool successorPinned, IPEndPoint? retiringEndPoint) { var watch = ValueStopwatch.StartNew(); // With less than the floor left of the window there is no hold worth having: the server is about to close @@ -631,14 +619,14 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su // one lifecycle for the protection: raised here with the hold, on each bridge (the FailFast shield) and on the // server (the timeout cover); lowered only when a replacement establishes, when the handoff gives up on a // bridge, or when this span runs out - never on a local idea of "done" along the way - interactiveBridge?.BeginQuiesce(cover); - subscriptionBridge?.BeginQuiesce(cover); + interactiveBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned); + subscriptionBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned); // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the // server closes it. Each bridge is released the moment its own connection is settled. - var interactiveTask = RetireBridgeAsync(interactiveBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned, retiringAddress); - var subscriptionTask = RetireBridgeAsync(subscriptionBridge, watch, cap, reason, retireCurrentIfTargetGone: !successorPinned, retiringAddress); + var interactiveTask = RetireBridgeAsync(interactiveBridge, watch, cap, reason); + var subscriptionTask = RetireBridgeAsync(subscriptionBridge, watch, cap, reason); var interactiveResult = await interactiveTask.ForAwait(); var subscriptionResult = await subscriptionTask.ForAwait(); @@ -666,7 +654,7 @@ private readonly struct BridgeRetirement(bool? recycled, bool forced) /// Retires one bridge's connection once drained, or at the cap regardless, and releases its hold; see /// . /// - private async Task RetireBridgeAsync(PhysicalBridge? bridge, ValueStopwatch watch, TimeSpan cap, string reason, bool retireCurrentIfTargetGone, IPAddress? retiringAddress) + private async Task RetireBridgeAsync(PhysicalBridge? bridge, ValueStopwatch watch, TimeSpan cap, string reason) { if (bridge is null) return new(false, false); bool forced = false; @@ -677,7 +665,7 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V { var force = watch.ElapsedMilliseconds >= cap.TotalMilliseconds; forced |= force; - var result = await bridge.TryRetireQuiesceTargetAsync(force, retireCurrentIfTargetGone, retiringAddress, reason, lockDeadline.Token).ForAwait(); + var result = await bridge.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); if (result is { } done) { // whether we replaced the connection or it went away by itself, a replacement is now on its way, @@ -762,20 +750,12 @@ private static Random RandomFor(ServerEndPoint server) => private void ExtendRelaxedWindow(TimeSpan duration, string cause) { - var candidate = NudgeFromZeroTicks(unchecked(Environment.TickCount + (int)duration.TotalMilliseconds)); - while (true) + // a new notification never shortens an existing window: a FAILING_OVER arriving inside a longer + // MIGRATING window must not cut it short + if (_relaxedDeadline.TryExtendBy((int)duration.TotalMilliseconds, out var deadline)) { - var current = Volatile.Read(ref _relaxedDeadlineTicks); - - // a new notification never shortens an existing window: a FAILING_OVER arriving inside a longer - // MIGRATING window must not cut it short - if (current != 0 && unchecked(candidate - current) <= 0) return; - if (Interlocked.CompareExchange(ref _relaxedDeadlineTicks, candidate, current) == current) - { - Volatile.Write(ref _relaxedEndedTicks, candidate); - Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); - return; - } + Volatile.Write(ref _relaxedEndedTicks, deadline); + Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); } } From 1924d9fcd87484c4701ec6be43a1595e97fee5ad Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 13:37:50 +0000 Subject: [PATCH 14/27] Handoff drain: a resolved retire makes the next connection the replacement; a hold that diverted nothing protects nothing Thirteenth round of review follow-ups: - The bona fide test gains the input it was missing: once the retire has resolved, whatever connection comes up next is the replacement wherever it landed, including the same address and port - the common case after an in-place proxy restart or with stale DNS - so the protection stands down on its establishment instead of outliving its deadline. Before the retire resolves, the landed-elsewhere test still tells a replacement from a reconnect that landed back on the retiring node. The answer is latched on the connection once true, so the hot path does not re-read the socket. - A hold records whether it diverted anything (or inherited an earlier hold's diverted work); a drain pass that finds no connection lowers the protection only if nothing was diverted, whether or not the hold had a target. - TickDeadline takes TimeSpans and clamps them, and the announced MOVING window is capped at an hour on entry, so an absurd window cannot overflow a delay, a cancellation source or a deadline into the past. - The handoff's relaxation respects maintRelaxedWindowMax, as the notification path does. - The retire's report and dispose run under try/finally: a throwing user handler neither leaks the connection nor makes the retire look undone. - The hold cover extends rather than overwrites, per its documented lifecycle. - QueueOrFailMessage is back to its original shape; the gates queue held work directly, a batch with one fence for all its members. --- .../Maintenance/TickDeadline.cs | 26 +++- src/StackExchange.Redis/PhysicalBridge.cs | 127 +++++++++++++----- src/StackExchange.Redis/PhysicalConnection.cs | 11 +- .../ServerEndPoint.Maintenance.cs | 22 ++- 4 files changed, 140 insertions(+), 46 deletions(-) diff --git a/src/StackExchange.Redis/Maintenance/TickDeadline.cs b/src/StackExchange.Redis/Maintenance/TickDeadline.cs index ec2256d59..b9842412d 100644 --- a/src/StackExchange.Redis/Maintenance/TickDeadline.cs +++ b/src/StackExchange.Redis/Maintenance/TickDeadline.cs @@ -15,22 +15,38 @@ internal struct TickDeadline { private int _ticks; + /// + /// The most a deadline can be set into the future: about six days. A tick count wraps in about 24.8 days, so a + /// span approaching that would be indistinguishable from the past; and a span that overflows an int of + /// milliseconds (a buggy or hostile announced window) would otherwise land the deadline in the past outright, + /// raising a protection that is silently inactive. No maintenance window is days long; clamping is always right. + /// + private const int MaxMilliseconds = int.MaxValue / 4; + /// The raw deadline, or zero when unset. For callers that record *when* a window was due to end. public int Ticks => Volatile.Read(ref _ticks); - public void SetIn(int milliseconds) => Volatile.Write(ref _ticks, NudgeFromZero(unchecked(Environment.TickCount + milliseconds))); + public void SetIn(TimeSpan span) => Volatile.Write(ref _ticks, NudgeFromZero(unchecked(Environment.TickCount + Clamp(span)))); + + /// A span as whole milliseconds, never negative and never beyond . + public static int Clamp(TimeSpan span) + { + if (span <= TimeSpan.Zero) return 0; + var ms = span.TotalMilliseconds; + return ms >= MaxMilliseconds ? MaxMilliseconds : (int)ms; + } public void Clear() => Volatile.Write(ref _ticks, 0); /// - /// Moves the deadline out to now plus unless it is already later than that. + /// Moves the deadline out to now plus unless it is already later than that. /// - /// How far from now the deadline should be, at least. + /// How far from now the deadline should be, at least. /// The deadline now in force, whether or not this call set it. /// Whether this call extended it. - public bool TryExtendBy(int milliseconds, out int deadline) + public bool TryExtendBy(TimeSpan span, out int deadline) { - var candidate = NudgeFromZero(unchecked(Environment.TickCount + milliseconds)); + var candidate = NudgeFromZero(unchecked(Environment.TickCount + Clamp(span))); while (true) { var current = Volatile.Read(ref _ticks); diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 5ed167b3f..19ef139db 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -133,7 +133,7 @@ public enum State : byte /// internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); - private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceInheritedProtection; + private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceInheritedProtection, _quiesceDiverted; private PhysicalConnection? _quiesceTarget; private System.Net.IPEndPoint? _quiesceRetiringEndPoint; @@ -158,20 +158,40 @@ internal void ClearHandoffPending() /// /// Whether a connection is a bona fide replacement for the one a handoff is retiring: not that connection - /// itself, and - where reconnection is not pinned to a named successor, so the ordinary reconnect may have - /// landed straight back on the node being retired - demonstrably somewhere else, by address *and* port, since - /// Redis Enterprise regularly rebinds to another port on the same host. A connection whose address cannot be - /// read yet is not known to be a replacement. Only a bona fide replacement establishing lowers the protection, - /// releases the hold on new work, and receives the held backlog. + /// itself, and either established after the retire resolved (then it is the replacement wherever it landed), + /// or - before that, where reconnection is not pinned to a named successor, so the ordinary reconnect may + /// have landed straight back on the node being retired - demonstrably somewhere else, by address *and* port, + /// since Redis Enterprise regularly rebinds to another port on the same host. A connection whose address + /// cannot be read yet is not known to be a replacement. Only a bona fide replacement establishing lowers the + /// protection, releases the hold on new work, and receives the held backlog. /// private bool IsBonaFideReplacement(PhysicalConnection connection) { + if (connection.KnownReplacement) return true; // stable once true; latched so the hot path does not re-derive it if (connection.WasHandoffTarget) return false; - if (_quiesceSuccessorPinned) return true; - var retiring = _quiesceRetiringEndPoint; - if (retiring is null) return true; // nothing known to compare against - var landedOn = RemoteEndPointOrNull(connection); - return landedOn is not null && !landedOn.Equals(retiring); + bool bonaFide; + if (_quiesceSuccessorPinned || _quiesceRecycled) + { + // pinned to a named successor, any other connection is the replacement; and once the retire has + // resolved, whatever comes up next is the replacement wherever it landed - including the same + // address and port, which is the common case after an in-place proxy restart or with stale DNS + bonaFide = true; + } + else + { + var retiring = _quiesceRetiringEndPoint; + if (retiring is null) + { + bonaFide = true; // nothing known to compare against + } + else + { + var landedOn = RemoteEndPointOrNull(connection); + bonaFide = landedOn is not null && !landedOn.Equals(retiring); + } + } + if (bonaFide) connection.MarkKnownReplacement(); + return bonaFide; } private static System.Net.IPEndPoint? RemoteEndPointOrNull(PhysicalConnection connection) @@ -228,7 +248,8 @@ internal void BeginQuiesce(TimeSpan protection, System.Net.IPEndPoint? retiringE // it, with that handoff's diverted work queued) extends it rather than replacing it, and remembers that // there is older work to protect even if this hold itself diverts nothing _quiesceInheritedProtection = _handoffProtection.IsActive; - _handoffProtection.TryExtendBy((int)protection.TotalMilliseconds, out _); + _quiesceDiverted = _quiesceInheritedProtection; // older work still queued counts as diverted work to protect + _handoffProtection.TryExtendBy(protection, out _); _quiescing = true; } @@ -341,11 +362,12 @@ private static bool ShouldHoldBatch(List messages) // faulted), or there was none when the hold began because it had already dropped. if (current is null) { - // Nothing to retire. Whatever the hold diverted before the target went stays protected for the - // reconnect that is coming; only a hold that never had a target, and inherited no earlier - // handoff's protection, can have nothing to protect. + // Nothing to retire. Whatever the hold diverted - off the target, or off a reconnect that + // landed back on the retiring node and then dropped too - stays protected for the reconnect + // that is coming; only a hold that diverted nothing (and inherited no earlier handoff's work) + // has nothing to protect. _quiesceRecycled = true; - if (target is null && !_quiesceInheritedProtection) ClearHandoffPending(); + if (!_quiesceDiverted) ClearHandoffPending(); return false; } if (IsBonaFideReplacement(current)) @@ -388,9 +410,27 @@ private static bool ShouldHoldBatch(List messages) // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first - // also performs the disconnect, so the Dispose below is cleanup. - toRetire.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); - toRetire.Dispose(); + // also performs the disconnect, so the Dispose below is cleanup. A user handler that throws must not + // leak the connection, nor make the retire look as if it had not happened. + try + { + toRetire.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + } + catch (Exception ex) + { + OnInternalError(ex); + } + finally + { + try + { + toRetire.Dispose(); + } + catch (Exception ex) + { + OnInternalError(ex); + } + } return true; } @@ -438,21 +478,15 @@ public void ReportNextFailure() public override string ToString() => ConnectionType + "/" + Format.ToString(ServerEndPoint.EndPoint); - private WriteResult QueueOrFailMessage(Message message, bool holdForHandoff = false) + private WriteResult QueueOrFailMessage(Message message) { - // If it's an internal call that's not a QUIT, or we're allowed to queue in general, then queue. Work held - // for a maintenance handoff is queued whatever the policy: the connection is live and the message would - // otherwise have been written, so failing it would be a regression (see ShouldHoldForHandoff). - if (holdForHandoff || message.IsInternalCall || Multiplexer.RawConfig.BacklogPolicy.QueueWhileDisconnected) + // If it's an internal call that's not a QUIT + // or we're allowed to queue in general, then queue + if (message.IsInternalCall || Multiplexer.RawConfig.BacklogPolicy.QueueWhileDisconnected) { // Let's just never ever queue a QUIT message if (message.Command != RedisCommand.QUIT) { - if (holdForHandoff) - { - EnqueueHeldAndWakeIfReleased(message); - return WriteResult.Success; - } message.SetEnqueued(null); BacklogEnqueue(message); // Note: we don't start a worker on each message here @@ -480,7 +514,11 @@ private WriteResult FailDueToNoConnection(Message message) public WriteResult TryWriteSync(Message message, bool isReplica) { if (isDisposed) throw new ObjectDisposedException(Name); - if (ShouldHoldForHandoff(message)) return QueueOrFailMessage(message, holdForHandoff: true); + if (ShouldHoldForHandoff(message)) + { + EnqueueHeldAndWakeIfReleased(message); + return WriteResult.Success; // queued counts as success + } if (!IsConnected || NeedsReconnect) return QueueOrFailMessage(message); var physical = this.physical; @@ -505,7 +543,11 @@ public WriteResult TryWriteSync(Message message, bool isReplica) public ValueTask TryWriteAsync(Message message, bool isReplica, bool bypassBacklog = false) { if (isDisposed) throw new ObjectDisposedException(Name); - if (!bypassBacklog && ShouldHoldForHandoff(message)) return new ValueTask(QueueOrFailMessage(message, holdForHandoff: true)); + if (!bypassBacklog && ShouldHoldForHandoff(message)) + { + EnqueueHeldAndWakeIfReleased(message); + return new ValueTask(WriteResult.Success); // queued counts as success + } if ((!IsConnected || NeedsReconnect) && !bypassBacklog) return new ValueTask(QueueOrFailMessage(message)); var physical = this.physical; @@ -1075,10 +1117,7 @@ internal bool TryEnqueue(List messages, bool isReplica) if (HoldActive && ShouldHoldBatch(messages)) { // a maintenance handoff is draining the connection: hold the batch for the replacement - foreach (var message in messages) - { - QueueOrFailMessage(message, holdForHandoff: true); - } + EnqueueHeldBatchAndWakeIfReleased(messages); return true; } @@ -1361,6 +1400,24 @@ private void EnqueueHeldAndWakeIfReleased(Message message, bool requeue = false) if (requeue) message.SetRequeued(); else message.SetEnqueued(null); BacklogEnqueue(message); + _quiesceDiverted = true; // there is now work the handoff's protection must stand for + WakeIfHoldReleased(); + } + + /// A batch held as a unit: every member queued, then one fence and one check for the batch. + private void EnqueueHeldBatchAndWakeIfReleased(List messages) + { + foreach (var message in messages) + { + message.SetEnqueued(null); + BacklogEnqueue(message); + } + _quiesceDiverted = true; + WakeIfHoldReleased(); + } + + private void WakeIfHoldReleased() + { Interlocked.MemoryBarrier(); if (!_quiescing) StartBacklogProcessor(); } diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index eea126ffb..ef2f9cf32 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -988,7 +988,16 @@ internal void GetStormLog(StringBuilder sb) // volatile: each is written on one thread and read on others as part of the handoff's identity test, in the // very races that test exists for; a plain field could be read stale exactly then - private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished; + private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished, _knownReplacement; + + /// + /// Latched once this connection has been found to be a bona fide replacement for one a maintenance handoff + /// retired (see PhysicalBridge.IsBonaFideReplacement): the answer is stable once true, and the test + /// behind it reads the socket's remote address, which is not something to repeat on every write. + /// + internal bool KnownReplacement => _knownReplacement; + + internal void MarkKnownReplacement() => _knownReplacement = true; /// /// Whether this connection was ever the one a maintenance handoff held work back from. A durable fact about the diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 5bad3a6a7..fa1890eee 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -228,9 +228,11 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) /// private void SetHoldCover(TimeSpan span) { - var ms = (int)span.TotalMilliseconds; + // extend, never shorten: a handoff beginning while an earlier one's cover is up must not cut that one short + var ms = TickDeadline.Clamp(span); + if (_holdCover.IsActive) ms = Math.Max(ms, Volatile.Read(ref _holdCoverMilliseconds)); Volatile.Write(ref _holdCoverMilliseconds, ms); - _holdCover.SetIn(ms); + _holdCover.TryExtendBy(span, out _); } /// @@ -327,7 +329,7 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? return; } - _relaxedDeadline.SetIn((int)tail.TotalMilliseconds); + _relaxedDeadline.SetIn(tail); Volatile.Write(ref _relaxedEndedTicks, _relaxedDeadline.Ticks); Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } @@ -432,6 +434,7 @@ internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalC var budget = window is { } value && value > TimeSpan.Zero ? value : Multiplexer.RawConfig.MaintenanceRelaxedTimeout; // no window given: use the relaxation floor + if (budget > MaxHandoffWindow) budget = MaxHandoffWindow; // an absurd announced window must not overflow a delay or a deadline IPEndPoint? current = null; try { @@ -558,6 +561,13 @@ private TimeSpan DrainCap(TimeSpan budget) /// How much longer than the cap the forced pass may wait for the write lock before giving the connection up to the server's close. private static readonly TimeSpan LockGrace = TimeSpan.FromSeconds(1); + /// + /// The longest MOVING window acted on. Measured windows are 15 seconds; an hour is already absurd, and anything + /// larger (a buggy or hostile announcement) would overflow the delays, cancellation sources and tick deadlines + /// downstream, or hold traffic for days. + /// + private static readonly TimeSpan MaxHandoffWindow = TimeSpan.FromHours(1); + /// /// Holds new caller work back, lets what has already been written finish, then replaces the connections. /// @@ -607,7 +617,9 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su // reconnect can take until the end of the window plus one connect attempt: a MoveTo pins the socket target // to the named successor for the rest of the window, and a successor that does not answer is tried until // that expires before the endpoint is resolved the ordinary way. - ExtendRelaxedWindow(cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout, "MOVING handoff"); + var relax = cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout; + if (relax > Multiplexer.RawConfig.MaintenanceRelaxedWindowMax) relax = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; // the user's ceiling applies here as on the notification path + ExtendRelaxedWindow(relax, "MOVING handoff"); var pinned = cap + LockGrace > budget ? cap + LockGrace : budget; var cover = pinned + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); SetHoldCover(cover); @@ -752,7 +764,7 @@ private void ExtendRelaxedWindow(TimeSpan duration, string cause) { // a new notification never shortens an existing window: a FAILING_OVER arriving inside a longer // MIGRATING window must not cut it short - if (_relaxedDeadline.TryExtendBy((int)duration.TotalMilliseconds, out var deadline)) + if (_relaxedDeadline.TryExtendBy(duration, out var deadline)) { Volatile.Write(ref _relaxedEndedTicks, deadline); Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); From c58abb9fde9139a4810cf502549a172ffc2dd90a Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 14:09:55 +0000 Subject: [PATCH 15/27] Handoff drain: target status dominates the replacement latch, the landed-on test applies after the retire, a lockless retire as the last resort Fourteenth round of review follow-ups: - A connection that an earlier handoff latched as its replacement and that a later handoff now retires is a target, not a replacement: marking it a target clears the latch, and the target check comes first. - With no successor pinned, the landed-on comparison applies after the retire as well as before it: a reconnect that landed back on the node being retired is not a replacement at any point. The hold keeps new work and the held backlog off it until a bona fide replacement is current or the protection expires, instead of flushing onto a socket the server is about to close; and the FailFast shield now covers any failure of a connection that is not a known, established replacement, which includes that one. The latch is taken only once a connection is established, so a still-connecting successor that fails still counts as the handoff's own. - The hold cover is bounded by maintRelaxedWindowMax, as the relaxation is, while always spanning the drain itself and one connect attempt. - When the write lock cannot be had for the whole of the drain's deadline, the target is retired without it as a last resort - what every recycle did before handoffs existed - rather than left to the server's window-end close, which would fail everything in flight to protect at most one wedged writer's message. - The guarded remote-endpoint read lives on PhysicalConnection; the inherited- protection flag is a local. --- src/StackExchange.Redis/PhysicalBridge.cs | 131 ++++++++++++------ src/StackExchange.Redis/PhysicalConnection.cs | 28 +++- .../ServerEndPoint.Maintenance.cs | 30 ++-- 3 files changed, 125 insertions(+), 64 deletions(-) diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 19ef139db..5068c540e 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -133,7 +133,7 @@ public enum State : byte /// internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); - private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceInheritedProtection, _quiesceDiverted; + private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceDiverted; private PhysicalConnection? _quiesceTarget; private System.Net.IPEndPoint? _quiesceRetiringEndPoint; @@ -156,67 +156,98 @@ internal void ClearHandoffPending() internal bool HandoffPending => _handoffProtection.IsActive; + /// + /// The last resort when the write lock could not be had for the whole of the drain's deadline: retires the + /// target without it. A writer wedged on the lock is the only thing this can race, and what it risks is that + /// one writer's message failing with a write fault; what it buys is the replacement starting now rather than + /// at the server's window-end close, which would fail everything in flight and leave the bridge dark for the + /// reconnect. Before handoffs existed the recycle was always lockless, so this is the old behaviour kept as + /// the fallback rather than the rule. + /// + internal bool RetireQuiesceTargetWithoutLock(string reason) + { + var target = _quiesceTarget; + if (target is null || _quiesceRecycled || !ReferenceEquals(physical, target)) return false; + Trace($"recycling {ConnectionType} connection without the write lock (last resort): {reason}"); + target.MarkRetiredForHandoff(); + _quiesceRecycled = true; + try + { + target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + } + catch (Exception ex) + { + OnInternalError(ex); + } + finally + { + try + { + target.Dispose(); + } + catch (Exception ex) + { + OnInternalError(ex); + } + } + return true; + } + /// /// Whether a connection is a bona fide replacement for the one a handoff is retiring: not that connection - /// itself, and either established after the retire resolved (then it is the replacement wherever it landed), - /// or - before that, where reconnection is not pinned to a named successor, so the ordinary reconnect may - /// have landed straight back on the node being retired - demonstrably somewhere else, by address *and* port, - /// since Redis Enterprise regularly rebinds to another port on the same host. A connection whose address + /// itself (target status dominates, even over an earlier handoff having latched it as a replacement), and - + /// where reconnection is not pinned to a named successor, so the ordinary reconnect may have landed straight + /// back on the node being retired, before or after our retire - demonstrably somewhere else, by address *and* + /// port, since Redis Enterprise regularly rebinds to another port on the same host. A connection whose address /// cannot be read yet is not known to be a replacement. Only a bona fide replacement establishing lowers the /// protection, releases the hold on new work, and receives the held backlog. /// private bool IsBonaFideReplacement(PhysicalConnection connection) { + if (connection.WasHandoffTarget) return false; // target status dominates, including over an earlier handoff's latch if (connection.KnownReplacement) return true; // stable once true; latched so the hot path does not re-derive it - if (connection.WasHandoffTarget) return false; bool bonaFide; - if (_quiesceSuccessorPinned || _quiesceRecycled) + if (_quiesceSuccessorPinned) { - // pinned to a named successor, any other connection is the replacement; and once the retire has - // resolved, whatever comes up next is the replacement wherever it landed - including the same - // address and port, which is the common case after an in-place proxy restart or with stale DNS - bonaFide = true; + bonaFide = true; // pinned to a named successor: any connection other than the target is the replacement } else { + // Before *and* after the retire: with no successor to pin to, DNS can keep naming the node being + // retired for 4 to 19 seconds (measured), so a reconnect - whether it replaced a target that died by + // itself or followed our own retire - may have landed straight back on it. Such a connection is not a + // replacement: the hold keeps new work and the held backlog off it rather than flushing onto a socket + // the server is about to close, and its close is the handoff's own failure. One whose address cannot be + // read yet is not classifiable yet, and is not latched either way. var retiring = _quiesceRetiringEndPoint; if (retiring is null) { - bonaFide = true; // nothing known to compare against + // nothing to compare against: the connection that was told had no readable address. Deferring + // forever would hold traffic until the protection expires, so the replacement is taken at its word + bonaFide = true; } else { - var landedOn = RemoteEndPointOrNull(connection); + var landedOn = connection.TryGetRemoteEndPoint(); bonaFide = landedOn is not null && !landedOn.Equals(retiring); } } - if (bonaFide) connection.MarkKnownReplacement(); + // latched only once established: a still-connecting successor that then fails must still count as the + // handoff's own failure, which the latch would deny + if (bonaFide && connection.EverFullyEstablished) connection.MarkKnownReplacement(); return bonaFide; } - private static System.Net.IPEndPoint? RemoteEndPointOrNull(PhysicalConnection connection) - { - try - { - return connection.VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint; - } - catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) - { - return null; // not connected yet, or already gone: unknown - } - } - /// - /// Whether a connection failure belongs to a maintenance handoff in progress. Two durable facts about the - /// connection decide it, not where the handoff's flow happens to stand: it was ever the connection a hold - /// diverted work off (retired by the drain, closed by the server at the end of the window, or dropped by - /// itself - its death must not abandon that work), or it never completed its handshake (an attempt to bring - /// up the replacement, whatever the failure type, since a reset mid-handshake reports as a socket failure like - /// any other). A replacement that was fully established and then failed is an ordinary failure, and a FailFast - /// policy applies to it as usual, even while the handoff's protection is still up. + /// Whether a connection failure belongs to a maintenance handoff in progress: while the protection is up, any + /// failure of a connection that is not a known, established replacement is the handoff's own - the target + /// (retired by the drain, closed by the server at the end of the window, or dropped by itself), a reconnect + /// that landed back on the node being retired, or an attempt to bring up the replacement that never completed + /// its handshake, whatever the failure type. Only a replacement that was fully established and then failed is + /// an ordinary failure, and a FailFast policy applies to it as usual, even while the protection is still up. /// private bool IsHandoffOwnFailure(PhysicalConnection connection) - => HandoffPending && (connection.WasHandoffTarget || !connection.EverFullyEstablished); + => HandoffPending && !connection.KnownReplacement; /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has @@ -247,8 +278,8 @@ internal void BeginQuiesce(TimeSpan protection, System.Net.IPEndPoint? retiringE // a hold that begins while an earlier handoff's protection is still up (the bridge still reconnecting from // it, with that handoff's diverted work queued) extends it rather than replacing it, and remembers that // there is older work to protect even if this hold itself diverts nothing - _quiesceInheritedProtection = _handoffProtection.IsActive; - _quiesceDiverted = _quiesceInheritedProtection; // older work still queued counts as diverted work to protect + var inheritedProtection = _handoffProtection.IsActive; + _quiesceDiverted = inheritedProtection; // older work still queued counts as diverted work to protect _handoffProtection.TryExtendBy(protection, out _); _quiescing = true; } @@ -273,14 +304,28 @@ internal void EndQuiesce() private bool ShouldHoldForHandoff(Message message) => HoldActive && IsHoldable(message); /// - /// A handoff's hold has not yet been resolved: the connection it began on has not been retired, and whatever is - /// current is not a bona fide replacement either (it may be the target still, or a reconnect that landed back - /// on the node being retired). While unresolved, new work is held and the backlog does not flush, because - /// whatever is current is about to be retired. The moment a bona fide replacement is current, or the retire has - /// happened, the hold is over and the backlog may flush to whatever is current. + /// A handoff's hold is in force: whatever is current is not a bona fide replacement - it is the target still, or + /// a reconnect that landed back on the node being retired - and the server is going to close it. While it + /// holds, new work is queued and the backlog does not flush, because writing to that connection would only + /// hand the server work to fail. The moment a bona fide replacement is current (or the protection has expired), + /// the hold is over and the backlog flushes to it. /// private bool HoldUnresolved - => _quiescing && !_quiesceRecycled && !(physical is { } current && !ReferenceEquals(current, _quiesceTarget) && IsBonaFideReplacement(current)); + { + get + { + var current = physical; + if (_quiescing && !_quiesceRecycled) + { + // the retire has not happened: hold unless a bona fide replacement is already current + return !(current is { } c && !ReferenceEquals(c, _quiesceTarget) && IsBonaFideReplacement(c)); + } + // after the retire (or after the drain let go), the protection may still be up and what is current may + // be a reconnect that landed back on the node being retired: nothing is flushed onto that, and new work + // is not written to it, until a bona fide replacement is current or the protection expires + return HandoffPending && current is { } c2 && !IsBonaFideReplacement(c2); + } + } /// A handoff's hold is unresolved, and the current connection is live. private bool HoldActive => HoldUnresolved && IsConnected && !NeedsReconnect; @@ -900,7 +945,7 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // ServerEndPoint.OnFullyEstablished lowers the shared cover once no bridge needs it): not the very // connection the hold is draining completing a handshake it was in the middle of, and not a reconnect // that landed back on the node being retired, which the drain is about to retire in its turn - if (IsBonaFideReplacement(connection)) _handoffProtection.Clear(); + if (IsBonaFideReplacement(connection)) _handoffProtection.Clear(); // latches, since the connection is established by now Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index ef2f9cf32..79cc70fbe 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -991,9 +991,11 @@ internal void GetStormLog(StringBuilder sb) private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished, _knownReplacement; /// - /// Latched once this connection has been found to be a bona fide replacement for one a maintenance handoff - /// retired (see PhysicalBridge.IsBonaFideReplacement): the answer is stable once true, and the test - /// behind it reads the socket's remote address, which is not something to repeat on every write. + /// Latched once this connection, fully established, has been found to be a bona fide replacement for one a + /// maintenance handoff retired (see PhysicalBridge.IsBonaFideReplacement): the answer is stable once + /// true, and the test behind it reads the socket's remote address, which is not something to repeat on every + /// write. Cleared if the connection later becomes a handoff's target itself. A failure of a connection that is + /// *not* a known replacement, while a handoff's protection is up, is the handoff's own. /// internal bool KnownReplacement => _knownReplacement; @@ -1007,7 +1009,25 @@ internal void GetStormLog(StringBuilder sb) /// internal bool WasHandoffTarget => _wasHandoffTarget; - internal void MarkHandoffTarget() => _wasHandoffTarget = true; + /// Target status dominates: a connection that was a replacement for an earlier handoff and is now the one being retired is a target, not a replacement. + internal void MarkHandoffTarget() + { + _knownReplacement = false; + _wasHandoffTarget = true; + } + + /// The socket's remote address and port, or null if the socket is not connected yet, is already gone, or there is no socket. + internal System.Net.IPEndPoint? TryGetRemoteEndPoint() + { + try + { + return VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint; + } + catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) + { + return null; + } + } /// /// Whether this connection ever completed its handshake. A failure on a connection that never did is a failed diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index fa1890eee..65705db7d 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -435,15 +435,7 @@ internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalC ? value : Multiplexer.RawConfig.MaintenanceRelaxedTimeout; // no window given: use the relaxation floor if (budget > MaxHandoffWindow) budget = MaxHandoffWindow; // an absurd announced window must not overflow a delay or a deadline - IPEndPoint? current = null; - try - { - current = connection.VolatileSocket?.RemoteEndPoint as IPEndPoint; - } - catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) - { - // the socket is gone or not connected: no address to compare against - } + var current = connection.TryGetRemoteEndPoint(); // null if the socket is gone or not connected: nothing to compare against _ = HandoffAsync(budget, successor, current); } @@ -620,8 +612,14 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su var relax = cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout; if (relax > Multiplexer.RawConfig.MaintenanceRelaxedWindowMax) relax = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; // the user's ceiling applies here as on the notification path ExtendRelaxedWindow(relax, "MOVING handoff"); + var drainSpan = cap + LockGrace + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); var pinned = cap + LockGrace > budget ? cap + LockGrace : budget; var cover = pinned + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); + // the user's ceiling on relaxation bounds the cover too - a huge announced window must not floor every timeout + // for its whole length - but the cover always spans at least the drain itself and one connect attempt + var ceiling = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; + if (ceiling < drainSpan) ceiling = drainSpan; + if (cover > ceiling) cover = ceiling; SetHoldCover(cover); // snapshots: Dispose nulls the fields, and a handoff in progress must not fault on a server going away @@ -686,14 +684,12 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V } if (lockDeadline.IsCancellationRequested) { - // The write lock could not be had even for the forced pass: something is stuck on it. Recycling - // without the lock would reopen the mid-write loss the lock exists to prevent, so the connection - // is left to the server's close, which is where things stood before handoffs existed. The hold on - // *new* work ends (the finally below); the protection stays up for the work already diverted, - // which was queued off a live connection whatever the policy on the promise of a replacement - - // the server's close and the reconnect after it are that replacement arriving late, and the - // protection lowers when it establishes or its span runs out, as always. - return new(null, forced); + // The write lock could not be had even for the forced pass: something is wedged on it. Leaving the + // connection to the server's window-end close would fail everything in flight and leave the bridge + // dark for the reconnect - the whole window's traffic, to protect at most one wedged writer's + // message from a write fault. So, as a last resort, retire without the lock, which is what every + // recycle did before handoffs existed. The protection stays up for the diverted work either way. + return new(bridge.RetireQuiesceTargetWithoutLock(reason) ? true : null, forced); } await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } From ee6ae39a5e6ef1c83bdc4e57f6ef9fe4ebe29ea4 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 14:44:15 +0000 Subject: [PATCH 16/27] Handoff drain: a temporal fallback where endpoint identity is meaningless; the cover applies to held commands only Fifteenth round of review follow-ups: - Behind a Tunnel, or when the configured endpoint is itself a fixed address in front of the cluster, every connection reports the intermediary's address, so comparing it with the retiring one would make the genuine replacement look like a return to the retiring node forever. In those topologies the discriminator is temporal: a connection that is not the target and came up after the retire resolved is the replacement. - The hold's timeout cover applies to the commands the hold diverted (and to those it requeued), not to every command on the server: they carry a mark, and the backlog sweep, the in-flight sweep and the sync wait consult an overload that adds the cover for them alone. An application that has turned relaxation off keeps its short timeouts for everything else; a held command at the head of a sweep can delay, never lose, the one behind it. The cover still spans the drain and one connect attempt, since that is what the held commands need. - The diverted flag is set before the enqueue, so a drain pass cannot see a held message with the flag still off. - The socket's remote endpoint is cached on the connection once read. - The handoff target's expiry is a TickDeadline too; it was the last hand-rolled tick deadline. - The fence-pairing comments name WakeIfHoldReleased, which every held enqueue goes through, as the single enqueue-side site. - The steady-load test cancels its workers on any exit. --- .../ConnectionMultiplexer.cs | 4 +- src/StackExchange.Redis/Message.cs | 12 +++++ src/StackExchange.Redis/PhysicalBridge.cs | 49 ++++++++++++++----- src/StackExchange.Redis/PhysicalConnection.cs | 29 ++++++++--- .../ServerEndPoint.Maintenance.cs | 45 ++++++++++++----- .../MaintenanceNotificationTests.cs | 7 +++ 6 files changed, 114 insertions(+), 32 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 49a5682e5..f692c8cfe 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2737,7 +2737,7 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u // its async neighbour is relaxed. It can shrink back too: when a window closes the // effective timeout drops to the configured value, and we stop at that. var startedAt = Environment.TickCount; - var remaining = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds; + var remaining = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds, message) ?? TimeoutMilliseconds; while (true) { if (!Monitor.Wait(source, remaining)) @@ -2751,7 +2751,7 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u Trace("Timely response to " + message); break; } - var budget = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds) ?? TimeoutMilliseconds; + var budget = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds, message) ?? TimeoutMilliseconds; remaining = budget - unchecked(Environment.TickCount - startedAt); if (remaining <= 0) { diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 055b361c9..cc0135f1c 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -905,6 +905,18 @@ internal void SetRequeued() _queuedStampSent = _queuedStampReceived = -1; } + /// + /// Whether a maintenance handoff diverted this message off a live connection into the backlog (or requeued it + /// after retiring the connection it was headed for). Such a message is waiting on the client's own choice, not + /// on the server, so the handoff's timeout cover applies to it - and only to it: an ordinary command on the same + /// server keeps its configured timeout. + /// + internal bool HeldForHandoff => _heldForHandoff; + + internal void MarkHeldForHandoff() => _heldForHandoff = true; + + private volatile bool _heldForHandoff; + private int _writeTickCount; public int GetWriteTime() => Volatile.Read(ref _writeTickCount); diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 5068c540e..24b1ad69f 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -133,7 +133,7 @@ public enum State : byte /// internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); - private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceDiverted; + private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceDiverted, _quiesceEndpointIdentityMeaningful; private PhysicalConnection? _quiesceTarget; private System.Net.IPEndPoint? _quiesceRetiringEndPoint; @@ -211,6 +211,13 @@ private bool IsBonaFideReplacement(PhysicalConnection connection) { bonaFide = true; // pinned to a named successor: any connection other than the target is the replacement } + else if (!_quiesceEndpointIdentityMeaningful) + { + // behind a tunnel or a fixed front address every connection reports the same remote endpoint, so where + // it landed cannot be read off the socket; the discriminator is temporal instead - a connection that + // is not the target and came up after the retire resolved is the replacement + bonaFide = _quiesceRecycled && !ReferenceEquals(connection, _quiesceTarget); + } else { // Before *and* after the retire: with no successor to pin to, DNS can keep naming the node being @@ -267,13 +274,16 @@ private bool IsHandoffOwnFailure(PhysicalConnection connection) /// that was told, if known; see . /// Whether reconnection is pinned to a named successor, in which case any /// connection other than the target is a replacement. - internal void BeginQuiesce(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, bool successorPinned) + /// Whether a socket's remote endpoint identifies the node it reached; + /// behind a tunnel or a fixed front address it does not, and only time tells a replacement apart. + internal void BeginQuiesce(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, bool successorPinned, bool endpointIdentityMeaningful) { var target = physical; target?.MarkHandoffTarget(); _quiesceTarget = target; _quiesceRetiringEndPoint = retiringEndPoint; _quiesceSuccessorPinned = successorPinned; + _quiesceEndpointIdentityMeaningful = endpointIdentityMeaningful; _quiesceRecycled = false; // a hold that begins while an earlier handoff's protection is still up (the bridge still reconnecting from // it, with that handoff's diverted work queued) extends it rather than replacing it, and remembers that @@ -289,8 +299,9 @@ internal void EndQuiesce() if (!_quiescing) return; _quiescing = false; _quiesceTarget = null; - // Store the flag, fence, then load the queue - the mirror image of EnqueueHeldAndWakeIfReleased, which is - // the only enqueue-side half of this pairing. StartBacklogProcessor wakes a processor that is already running. + // Store the flag, fence, then load the queue - the mirror image of WakeIfHoldReleased, which every held + // enqueue (single, batch, requeue) goes through and which is the only enqueue-side half of this pairing. + // StartBacklogProcessor wakes a processor that is already running. Interlocked.MemoryBarrier(); if (BacklogHasItems) StartBacklogProcessor(); } @@ -1425,11 +1436,11 @@ private bool AnyCallerFacingQueued() /// /// Queues work held for a maintenance handoff - diverted at the gate, or requeued by a writer that reached the - /// lock to find its connection retired - and makes sure it is not stranded if the hold is released - /// concurrently: the queue is stored, then a full fence, then the flag is loaded - the mirror image of - /// , which stores the flag, fences, then loads the queue. With a full fence on both - /// sides at least one of them sees the other's store, so one side or the other starts the processor; neither - /// starts one for an empty backlog. This is the only place the enqueue side of that pairing lives. + /// lock to find its connection retired - and, through , makes sure it is not + /// stranded if the hold is released concurrently: the queue is stored, then a full fence, then the flag is + /// loaded - the mirror image of , which stores the flag, fences, then loads the queue. + /// With a full fence on both sides at least one of them sees the other's store, so one side or the other starts + /// the processor; neither starts one for an empty backlog. /// /// /// A requeued message is appended, not inserted at the head: a head insert would buy only a best effort at the @@ -1442,25 +1453,33 @@ private bool AnyCallerFacingQueued() /// it keeps its write time, since the budget it has spent still counts, and only forgets that connection. private void EnqueueHeldAndWakeIfReleased(Message message, bool requeue = false) { + // before the enqueue, not after: a drain pass deciding whether there is anything to protect must not see + // the message in the backlog and the flag still off (it is monotonic within a hold, so early is harmless) + _quiesceDiverted = true; + message.MarkHeldForHandoff(); if (requeue) message.SetRequeued(); else message.SetEnqueued(null); BacklogEnqueue(message); - _quiesceDiverted = true; // there is now work the handoff's protection must stand for WakeIfHoldReleased(); } /// A batch held as a unit: every member queued, then one fence and one check for the batch. private void EnqueueHeldBatchAndWakeIfReleased(List messages) { + _quiesceDiverted = true; foreach (var message in messages) { + message.MarkHeldForHandoff(); message.SetEnqueued(null); BacklogEnqueue(message); } - _quiesceDiverted = true; WakeIfHoldReleased(); } + /// + /// The enqueue side of the pairing with , and the only one: the queue has been stored + /// (by whichever helper called this), then a full fence here, then the flag is loaded. + /// private void WakeIfHoldReleased() { Interlocked.MemoryBarrier(); @@ -1547,17 +1566,21 @@ private void CheckBacklogForTimeouts() // Because peeking at the backlog, checking message and then dequeuing, is not thread-safe, we do have to use // a lock here, for mutual exclusion of backlog DEQUEUERS. Unfortunately. // But we reduce contention by only locking if we see something that looks timed out. + // a command a maintenance handoff diverted here is covered by the handoff for as long as that stands; an + // ordinary command keeps the timeout above (a held one at the head delays, never loses, the one behind it) + int TimeoutFor(Message m) => m.HeldForHandoff ? ServerEndPoint.GetEffectiveTimeoutMilliseconds(Multiplexer.TimeoutMilliseconds, m) : timeout; + while (_backlog.TryPeek(out Message? message)) { // See if the message has pass our async timeout threshold // Note: All timed out messages must be dequeued, even when no completion is needed, to be able to dequeue and complete other timed out messages. - if (!message.HasTimedOut(now, timeout, out var _)) break; // not a timeout - we can stop looking + if (!message.HasTimedOut(now, TimeoutFor(message), out var _)) break; // not a timeout - we can stop looking lock (_backlog) { // Peek again since we didn't have lock before... // and rerun the exact same checks as above, note that it may be a different message now if (!_backlog.TryPeek(out message)) break; - if (!message.HasTimedOut(now, timeout, out var _)) break; + if (!message.HasTimedOut(now, TimeoutFor(message), out var _)) break; if (!BacklogTryDequeue(out var message2) || (message != message2)) // consume it for real { diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 79cc70fbe..5052360fe 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -1016,19 +1016,29 @@ internal void MarkHandoffTarget() _wasHandoffTarget = true; } - /// The socket's remote address and port, or null if the socket is not connected yet, is already gone, or there is no socket. + /// + /// The socket's remote address and port, or null if the socket is not connected yet, is already gone, or + /// there is no socket. Cached once read: a connected socket's remote endpoint never changes, and the handoff's + /// bona fide test asks on every write while a hold is unresolved. + /// internal System.Net.IPEndPoint? TryGetRemoteEndPoint() { + var cached = _remoteEndPoint; + if (cached is not null) return cached; try { - return VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint; + cached = VolatileSocket?.RemoteEndPoint as System.Net.IPEndPoint; } catch (Exception ex) when (ex is System.Net.Sockets.SocketException or ObjectDisposedException or InvalidOperationException) { return null; } + if (cached is not null) _remoteEndPoint = cached; + return cached; } + private volatile System.Net.IPEndPoint? _remoteEndPoint; + /// /// Whether this connection ever completed its handshake. A failure on a connection that never did is a failed /// attempt to bring a connection up, which is how a maintenance handoff tells "the replacement is not up yet" @@ -1036,7 +1046,11 @@ internal void MarkHandoffTarget() /// internal bool EverFullyEstablished => _everFullyEstablished; - internal void MarkFullyEstablished() => _everFullyEstablished = true; + internal void MarkFullyEstablished() + { + _everFullyEstablished = true; + _ = TryGetRemoteEndPoint(); // cache it now, while the socket is certainly connected + } /// /// Whether any message awaiting a response was issued by a *caller* rather than by us. @@ -1088,14 +1102,17 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou { // We only handle async timeouts here, synchronous timeouts are handled upstream. // Those sync timeouts happen in ConnectionMultiplexer.ExecuteSyncImpl() via Monitor.Wait. - if (msg.HasTimedOut(now, timeout, out var elapsed)) + // A command a maintenance handoff held, now written to the replacement, is covered by the + // handoff for as long as that stands; every other command keeps the timeout above. + var effective = msg.HeldForHandoff ? server.GetEffectiveTimeoutMilliseconds(multiplexer.AsyncTimeoutMilliseconds, msg) : timeout; + if (msg.HasTimedOut(now, effective, out var elapsed)) { if (msg.ResultBoxIsAsync) { bool haveDeltas = msg.TryGetPhysicalState(out _, out _, out long sentDelta, out var receivedDelta) && sentDelta >= 0 && receivedDelta >= 0; var baseErrorMessage = haveDeltas - ? $"Timeout awaiting response (outbound={sentDelta >> 10}KiB, inbound={receivedDelta >> 10}KiB, {elapsed}ms elapsed, timeout is {timeout}ms)" - : $"Timeout awaiting response ({elapsed}ms elapsed, timeout is {timeout}ms)"; + ? $"Timeout awaiting response (outbound={sentDelta >> 10}KiB, inbound={receivedDelta >> 10}KiB, {elapsed}ms elapsed, timeout is {effective}ms)" + : $"Timeout awaiting response ({elapsed}ms elapsed, timeout is {effective}ms)"; var timeoutEx = ExceptionFactory.Timeout(multiplexer, baseErrorMessage, msg, server); multiplexer.OnMessageFaulted(msg, timeoutEx); msg.SetExceptionAndComplete(timeoutEx, this); // tell the message that it is doomed diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 65705db7d..0f1c256bd 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -206,9 +206,23 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) } // a handoff hold diverts commands off a live connection into the backlog; whatever the relaxation setting - // (zero disables it), those commands must not be timed out by the hold itself, so the hold imposes its - // own floor on the effective timeout for as long as it, and the reconnect after it, can take - if (_holdCover.IsActive) + // (zero disables it), those commands must not be timed out by the hold itself - see the overload taking + // the message, which applies the hold's cover to them and to them alone + return result; + } + + /// + /// The effective timeout for a particular command: as , and + /// for a command that a maintenance handoff diverted into the backlog () no + /// less than the handoff's cover for as long as that stands. Only such a command: an ordinary command on the same + /// server, including one issued by an application that has turned relaxation off and runs short timeouts, keeps + /// its configured timeout. The sweeps stop at the first command that has not timed out, so a held command at the + /// head can delay (never lose) the timeout of an ordinary one behind it, by at most the cover. + /// + internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds, Message message) + { + var result = GetEffectiveTimeoutMilliseconds(configuredMilliseconds); + if (message.HeldForHandoff && _holdCover.IsActive) { var floor = Volatile.Read(ref _holdCoverMilliseconds); if (floor > result) result = floor; @@ -335,7 +349,7 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? } private volatile EndPoint? _handoffTarget; - private int _handoffTargetExpiryTicks; + private TickDeadline _handoffTargetExpiry; private int _handoffInFlight, _handoffRecycles; /// @@ -362,7 +376,7 @@ internal EndPoint? HandoffTarget var target = _handoffTarget; if (target is null) return null; - if (unchecked(Environment.TickCount - Volatile.Read(ref _handoffTargetExpiryTicks)) >= 0) + if (!_handoffTargetExpiry.IsActive) { _handoffTarget = null; // expired; resolve the endpoint the usual way from here on return null; @@ -377,7 +391,7 @@ internal EndPoint? HandoffTarget /// internal void SetHandoffTarget(EndPoint target, TimeSpan window) { - Volatile.Write(ref _handoffTargetExpiryTicks, unchecked(Environment.TickCount + (int)Math.Max(window.TotalMilliseconds, 1000))); + _handoffTargetExpiry.SetIn(window < TimeSpan.FromSeconds(1) ? TimeSpan.FromSeconds(1) : window); _handoffTarget = target; } @@ -615,8 +629,10 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su var drainSpan = cap + LockGrace + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); var pinned = cap + LockGrace > budget ? cap + LockGrace : budget; var cover = pinned + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); - // the user's ceiling on relaxation bounds the cover too - a huge announced window must not floor every timeout - // for its whole length - but the cover always spans at least the drain itself and one connect attempt + // the user's ceiling on relaxation bounds the cover too - a huge announced window must not floor the held + // commands' timeouts for its whole length - but the cover always spans at least the drain itself and one + // connect attempt, which is what the held commands need; it applies to them alone, so an application that + // has turned relaxation off keeps its short timeouts for everything else var ceiling = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; if (ceiling < drainSpan) ceiling = drainSpan; if (cover > ceiling) cover = ceiling; @@ -629,8 +645,15 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su // one lifecycle for the protection: raised here with the hold, on each bridge (the FailFast shield) and on the // server (the timeout cover); lowered only when a replacement establishes, when the handoff gives up on a // bridge, or when this span runs out - never on a local idea of "done" along the way - interactiveBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned); - subscriptionBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned); + // Whether a socket's remote endpoint says anything about *which node* it reached. Behind a Tunnel, or when + // the configured endpoint is itself a fixed address (a load balancer or VIP in front of the cluster), every + // connection reports the intermediary's address, and comparing it with the retiring one would make the + // genuine replacement look like a return to the retiring node forever. Then only time can tell: a connection + // established after the retire resolved is the replacement. + var identityMeaningful = Multiplexer.RawConfig.Tunnel is null + && !(EndPoint is IPEndPoint configured && retiringEndPoint is not null && configured.Equals(retiringEndPoint)); + interactiveBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned, identityMeaningful); + subscriptionBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned, identityMeaningful); // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the @@ -798,7 +821,7 @@ private void ExtendRelaxedWindow(TimeSpan duration, string cause) /// So a window that closed no longer ago than the command could have been waiting still counts. That is /// the tightest bound that catches every genuine case, and it is deliberately a bound rather than a /// certainty: without per-message state - which the timeout sweeps rule out, see - /// - a client cannot know whether *this* command overlapped + /// - a client cannot know whether *this* command overlapped /// the window, only whether it could have. /// /// diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 9fb9994bc..b3fae000e 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -887,6 +887,8 @@ private async Task SteadyLoadHandoffAsync(Action? configur } } })).ToArray(); + try + { Assert.True(await Poll.UntilAsync(() => Volatile.Read(ref completed) > 20), "the load should be flowing before the handoff"); var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); @@ -923,6 +925,11 @@ await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutM Assert.Empty(failures.Select(f => f.Message)); } Assert.DoesNotContain(logs.Lines, l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); + } + finally + { + stop.Cancel(); // on any exit: a failed assertion must not leave four workers hot-looping on a disposed multiplexer + } } } From 2179c1056dbbf1f8db16a62e0db5c8ffab268d72 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 16:27:39 +0000 Subject: [PATCH 17/27] Handoff drain: three-way classification with unknown deferring, the cover as a property of held commands, trust validated at use Sixteenth round of review follow-ups. The protection state machine's rules are now stated once at the top of the classification section, and the code is checked against them: - Classification has three outcomes: replacement, doomed, unknown. Unknown (an address not yet readable; or, behind a tunnel, time not yet telling) defers to the next pass and is never retired, never branded a target, and never trusted. - A successor pin is trusted only while it is still in force: the first bridge's replacement clears the pin, after which the other bridge's retry resolves the endpoint again and is classified by where it landed. - Behind a tunnel or a fixed front address, the temporal fallback also accepts "the drain has let go": after EndQuiesce nothing else will arrive, so what establishes is the replacement, rather than holding until the protection expires. - The timeout cover is a property of each held command for as long as it lives, no longer a window on the server, so it cannot be withdrawn between the replacement establishing and the held work being written to it; the FailFast shield likewise stands down only once the last held message has been flushed to the replacement. - A message that parks in a backlog that is holding (it passed the gate just before the hold rose) is held too, with the same cover. - The sync wait polls in one-second slices, so a relaxation or a cover that begins after a caller parked is noticed even when nothing pulses. - One teardown routine for the locked and lockless retire paths. The in-process test server can delay replies to one command only, so a test can slow the workload without pushing the replacement's own handshake past a short timeout; a new steady-load variant runs with relaxation off and a timeout shorter than a held command's wait. --- docs/ServerMaintenanceEvent.md | 2 +- .../ConnectionMultiplexer.cs | 19 +- src/StackExchange.Redis/PhysicalBridge.cs | 238 ++++++++++-------- .../ServerEndPoint.Maintenance.cs | 47 ++-- src/StackExchange.Redis/ServerEndPoint.cs | 4 - .../InProcessTestServer.cs | 17 +- .../MaintenanceNotificationTests.cs | 30 ++- 7 files changed, 202 insertions(+), 155 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 7ee167b90..8df500282 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. Until the replacement connection is established (or, if it never is, for the rest of the window plus one connect attempt), commands against that server time out no sooner than that whole span, whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; ordinary timeouts resume the moment every connection to that server has its replacement up. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. The commands the hold diverted keep a longer timeout until they are answered - the hold, the rest of the window and one connect attempt, bounded by `maintRelaxedWindowMax` - whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; every other command on that server keeps its configured timeout throughout, including under an application that has turned relaxation off. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index f692c8cfe..b724361bd 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2736,29 +2736,28 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u // sync caller in flight when a MIGRATING arrives times out at the strict timeout while // its async neighbour is relaxed. It can shrink back too: when a window closes the // effective timeout drops to the configured value, and we stop at that. + // + // Waiting in slices is what makes that re-read reachable: a single wait for the whole budget + // that nothing pulses (a genuinely slow command) would time out at the original deadline and + // never see a relaxation, or a handoff's cover, that began after we parked. + const int sliceMilliseconds = 1000; var startedAt = Environment.TickCount; - var remaining = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds, message) ?? TimeoutMilliseconds; while (true) { - if (!Monitor.Wait(source, remaining)) + var budget = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds, message) ?? TimeoutMilliseconds; + var remaining = budget - unchecked(Environment.TickCount - startedAt); + if (remaining <= 0) { Trace("Timeout performing " + message); timeout = true; break; } + Monitor.Wait(source, remaining < sliceMilliseconds ? remaining : sliceMilliseconds); if (source.IsCompleted) { Trace("Timely response to " + message); break; } - var budget = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds, message) ?? TimeoutMilliseconds; - remaining = budget - unchecked(Environment.TickCount - startedAt); - if (remaining <= 0) - { - Trace("Timeout performing " + message); - timeout = true; - break; - } } } } diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 24b1ad69f..4be595fae 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -133,7 +133,7 @@ public enum State : byte /// internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); - private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceDiverted, _quiesceEndpointIdentityMeaningful; + private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceDiverted, _quiesceEndpointIdentityMeaningful, _lowerProtectionAfterFlush; private PhysicalConnection? _quiesceTarget; private System.Net.IPEndPoint? _quiesceRetiringEndPoint; @@ -148,11 +148,7 @@ public enum State : byte private Maintenance.TickDeadline _handoffProtection; /// The handoff has nothing left to protect on this bridge: its protection stands down. - internal void ClearHandoffPending() - { - _handoffProtection.Clear(); - ServerEndPoint.OnHandoffProtectionLowered(); - } + internal void ClearHandoffPending() => _handoffProtection.Clear(); internal bool HandoffPending => _handoffProtection.IsActive; @@ -168,81 +164,89 @@ internal bool RetireQuiesceTargetWithoutLock(string reason) { var target = _quiesceTarget; if (target is null || _quiesceRecycled || !ReferenceEquals(physical, target)) return false; - Trace($"recycling {ConnectionType} connection without the write lock (last resort): {reason}"); target.MarkRetiredForHandoff(); _quiesceRecycled = true; - try - { - target.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); - } - catch (Exception ex) - { - OnInternalError(ex); - } - finally - { - try - { - target.Dispose(); - } - catch (Exception ex) - { - OnInternalError(ex); - } - } + RetireConnection(target, reason + " (without the write lock: last resort)"); return true; } /// - /// Whether a connection is a bona fide replacement for the one a handoff is retiring: not that connection - /// itself (target status dominates, even over an earlier handoff having latched it as a replacement), and - - /// where reconnection is not pinned to a named successor, so the ordinary reconnect may have landed straight - /// back on the node being retired, before or after our retire - demonstrably somewhere else, by address *and* - /// port, since Redis Enterprise regularly rebinds to another port on the same host. A connection whose address - /// cannot be read yet is not known to be a replacement. Only a bona fide replacement establishing lowers the - /// protection, releases the hold on new work, and receives the held backlog. + /// The rules the handoff's protection state machine keeps, stated once so that the next edit is checked + /// against them rather than against the last edit: + /// + /// (a) protection stands down only when the held work is safe - written to a bona fide + /// replacement, or carrying its own cover - never merely because a replacement exists; + /// (b) classifying a connection has three outcomes, replacement, doomed or unknown, and unknown always + /// defers to a later pass - it is never acted on as either of the others; + /// (c) every trust snapshot taken at - the pin, the retiring endpoint, + /// whether endpoint identity is meaningful - is validated where it is used, not assumed from capture. + /// + /// + private enum Replacement + { + /// A bona fide replacement: not the target, and demonstrably not back on the node being retired. + Yes, + + /// The target, or a reconnect that landed back on the node being retired: the server will close it. + Doomed, + + /// Not classifiable yet (its address cannot be read, or only time can tell and it has not yet): defer. + Unknown, + } + + /// + /// Classifies a connection against the handoff in progress. Target status dominates, including over an earlier + /// handoff's latch. A successor pin is trusted only while it is still in force (rule c): the first bridge's + /// replacement establishing clears the pin, after which the other bridge's retry resolves the endpoint again + /// and may land anywhere. Where endpoint identity is meaningless (a tunnel, a fixed front address) only time + /// tells: once the retire has resolved or the drain has let go, nothing else will arrive, so what is current is + /// the replacement; until then, unknown. Otherwise the landed-on endpoint decides, before and after the + /// retire, with an unreadable address meaning unknown. A verdict of replacement is latched on an established + /// connection so the hot path does not re-derive it. /// - private bool IsBonaFideReplacement(PhysicalConnection connection) + private Replacement Classify(PhysicalConnection connection) { - if (connection.WasHandoffTarget) return false; // target status dominates, including over an earlier handoff's latch - if (connection.KnownReplacement) return true; // stable once true; latched so the hot path does not re-derive it - bool bonaFide; - if (_quiesceSuccessorPinned) + if (connection.WasHandoffTarget) return Replacement.Doomed; + if (connection.KnownReplacement) return Replacement.Yes; + Replacement verdict; + if (_quiesceSuccessorPinned && ServerEndPoint.HandoffTarget is not null) { - bonaFide = true; // pinned to a named successor: any connection other than the target is the replacement + verdict = Replacement.Yes; } else if (!_quiesceEndpointIdentityMeaningful) { - // behind a tunnel or a fixed front address every connection reports the same remote endpoint, so where - // it landed cannot be read off the socket; the discriminator is temporal instead - a connection that - // is not the target and came up after the retire resolved is the replacement - bonaFide = _quiesceRecycled && !ReferenceEquals(connection, _quiesceTarget); + verdict = _quiesceRecycled || !_quiescing ? Replacement.Yes : Replacement.Unknown; } else { - // Before *and* after the retire: with no successor to pin to, DNS can keep naming the node being - // retired for 4 to 19 seconds (measured), so a reconnect - whether it replaced a target that died by - // itself or followed our own retire - may have landed straight back on it. Such a connection is not a - // replacement: the hold keeps new work and the held backlog off it rather than flushing onto a socket - // the server is about to close, and its close is the handoff's own failure. One whose address cannot be - // read yet is not classifiable yet, and is not latched either way. var retiring = _quiesceRetiringEndPoint; if (retiring is null) { - // nothing to compare against: the connection that was told had no readable address. Deferring - // forever would hold traffic until the protection expires, so the replacement is taken at its word - bonaFide = true; + // nothing to compare against: the connection that was told had no readable address. Deferring would + // hold traffic until the protection expired with nothing ever able to classify, so the replacement + // is taken at its word + verdict = Replacement.Yes; } else { var landedOn = connection.TryGetRemoteEndPoint(); - bonaFide = landedOn is not null && !landedOn.Equals(retiring); + if (landedOn is null) verdict = Replacement.Unknown; + else if (landedOn.Equals(retiring)) verdict = Replacement.Doomed; + else verdict = Replacement.Yes; } } - // latched only once established: a still-connecting successor that then fails must still count as the - // handoff's own failure, which the latch would deny - if (bonaFide && connection.EverFullyEstablished) connection.MarkKnownReplacement(); - return bonaFide; + if (verdict == Replacement.Yes && connection.EverFullyEstablished) connection.MarkKnownReplacement(); + return verdict; + } + + /// Whether the backlog still holds work a handoff diverted; the protection stands until it has been written (rule a). + private bool AnyHeldQueued() + { + foreach (var message in _backlog) + { + if (message.HeldForHandoff) return true; + } + return false; } /// @@ -271,7 +275,7 @@ private bool IsHandoffOwnFailure(PhysicalConnection connection) /// For how long at most the handoff's protection stands - the FailFast shield here and /// the timeout cover on the server - if no bona fide replacement establishes first. /// The address and port of the node being retired, as seen from the connection - /// that was told, if known; see . + /// that was told, if known; see . /// Whether reconnection is pinned to a named successor, in which case any /// connection other than the target is a replacement. /// Whether a socket's remote endpoint identifies the node it reached; @@ -290,6 +294,7 @@ internal void BeginQuiesce(TimeSpan protection, System.Net.IPEndPoint? retiringE // there is older work to protect even if this hold itself diverts nothing var inheritedProtection = _handoffProtection.IsActive; _quiesceDiverted = inheritedProtection; // older work still queued counts as diverted work to protect + _lowerProtectionAfterFlush = false; _handoffProtection.TryExtendBy(protection, out _); _quiescing = true; } @@ -315,26 +320,20 @@ internal void EndQuiesce() private bool ShouldHoldForHandoff(Message message) => HoldActive && IsHoldable(message); /// - /// A handoff's hold is in force: whatever is current is not a bona fide replacement - it is the target still, or - /// a reconnect that landed back on the node being retired - and the server is going to close it. While it - /// holds, new work is queued and the backlog does not flush, because writing to that connection would only - /// hand the server work to fail. The moment a bona fide replacement is current (or the protection has expired), - /// the hold is over and the backlog flushes to it. + /// A handoff's hold is in force: whatever is current is not known to be a bona fide replacement - it is the + /// target still, a reconnect that landed back on the node being retired, or one not classifiable yet - and + /// the server is going to close it, or might. While it holds, new work is queued and the backlog does not + /// flush, because writing to that connection would only hand the server work to fail. The moment a bona fide + /// replacement is current (or the protection has expired), the hold is over and the backlog flushes to it. /// private bool HoldUnresolved { get { + if (!((_quiescing && !_quiesceRecycled) || HandoffPending)) return false; var current = physical; - if (_quiescing && !_quiesceRecycled) - { - // the retire has not happened: hold unless a bona fide replacement is already current - return !(current is { } c && !ReferenceEquals(c, _quiesceTarget) && IsBonaFideReplacement(c)); - } - // after the retire (or after the drain let go), the protection may still be up and what is current may - // be a reconnect that landed back on the node being retired: nothing is flushed onto that, and new work - // is not written to it, until a bona fide replacement is current or the protection expires - return HandoffPending && current is { } c2 && !IsBonaFideReplacement(c2); + if (current is null || ReferenceEquals(current, _quiesceTarget)) return true; + return Classify(current) != Replacement.Yes; } } @@ -418,28 +417,38 @@ private static bool ShouldHoldBatch(List messages) // faulted), or there was none when the hold began because it had already dropped. if (current is null) { - // Nothing to retire. Whatever the hold diverted - off the target, or off a reconnect that - // landed back on the retiring node and then dropped too - stays protected for the reconnect - // that is coming; only a hold that diverted nothing (and inherited no earlier handoff's work) - // has nothing to protect. + // Nothing to retire. Whatever the hold diverted stays protected for the reconnect that is + // coming; only a hold that diverted nothing (and inherited no earlier handoff's work) has + // nothing to protect. _quiesceRecycled = true; if (!_quiesceDiverted) ClearHandoffPending(); return false; } - if (IsBonaFideReplacement(current)) + switch (Classify(current)) { - // the reconnect went somewhere else (a pinned successor, or DNS had already moved): it is the - // replacement, and retiring it would only force a second connect cycle. If it is already up, the - // protection stands down now; if it is still connecting, its establishment will do that. - _quiesceRecycled = true; - if (current.EverFullyEstablished) ClearHandoffPending(); - return false; + case Replacement.Yes: + // the reconnect is the replacement (a pinned successor, or demonstrably another node): + // retiring it would only force a second connect cycle. If it is already up, the + // protection stands down once the held work has reached it; if it is still connecting, + // its establishment will see to that. + _quiesceRecycled = true; + if (current.EverFullyEstablished) + { + if (AnyHeldQueued()) _lowerProtectionAfterFlush = true; + else ClearHandoffPending(); + } + return false; + case Replacement.Unknown: + // not classifiable yet - a socket still connecting, say (rule b): never retired on a guess, + // never branded a target; the next pass looks again, and the drain's deadline bounds this + return null; + default: + // back on the node being retired: what is current now is what the handoff replaces + toRetire = current; + current.MarkHandoffTarget(); + _quiesceTarget = current; + break; } - // back on the node being retired (or not yet known to be anywhere else): what is current now is - // what the handoff replaces - toRetire = current; - current.MarkHandoffTarget(); - _quiesceTarget = current; } if (!force && toRetire.HasCallerMessagesAwaitingResponse()) return null; @@ -461,16 +470,23 @@ private static bool ShouldHoldBatch(List messages) } } - Trace($"recycling {ConnectionType} connection: {reason}"); + RetireConnection(toRetire, reason); + return true; + } - // Report *before* tearing down, and in this order for a reason: RecordConnectionFailed only raises - // the public event while the pipe is still live ("if *we* didn't burn the pipe: flag it"), and - // Dispose runs Shutdown first - which is exactly why an ordinary dispose is silent. Recording first - // also performs the disconnect, so the Dispose below is cleanup. A user handler that throws must not - // leak the connection, nor make the retire look as if it had not happened. + /// + /// Tears down a connection a handoff has marked retired. Report before tearing down, and in this + /// order for a reason: RecordConnectionFailed only raises the public event while the pipe is still live + /// ("if *we* didn't burn the pipe: flag it"), and Dispose runs Shutdown first - which is exactly + /// why an ordinary dispose is silent. Recording first also performs the disconnect, so the dispose is cleanup. + /// A user handler that throws must not leak the connection, nor make the retire look as if it had not happened. + /// + private void RetireConnection(PhysicalConnection connection, string reason) + { + Trace($"recycling {ConnectionType} connection: {reason}"); try { - toRetire.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); + connection.RecordConnectionFailed(ConnectionFailureType.MaintenanceHandoff); } catch (Exception ex) { @@ -480,14 +496,13 @@ private static bool ShouldHoldBatch(List messages) { try { - toRetire.Dispose(); + connection.Dispose(); } catch (Exception ex) { OnInternalError(ex); } } - return true; } public void Dispose() @@ -952,11 +967,16 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) { reportNextFailure = reconfigureNextFailure = true; LastException = null; - // only a bona fide replacement establishing lowers the protection for this bridge (and then - // ServerEndPoint.OnFullyEstablished lowers the shared cover once no bridge needs it): not the very - // connection the hold is draining completing a handshake it was in the middle of, and not a reconnect - // that landed back on the node being retired, which the drain is about to retire in its turn - if (IsBonaFideReplacement(connection)) _handoffProtection.Clear(); // latches, since the connection is established by now + // only a bona fide replacement establishing lowers the protection for this bridge - not the very + // connection the hold is draining completing a handshake it was in the middle of when the hold began, + // and not a reconnect that landed back on the node being retired - and even then only once the work + // the hold diverted has been written to it (rule a): until the backlog processor has flushed the held + // messages, a failure would still abandon them under a FailFast policy + if (Classify(connection) == Replacement.Yes) + { + if (AnyHeldQueued()) _lowerProtectionAfterFlush = true; + else _handoffProtection.Clear(); + } Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -1373,6 +1393,13 @@ private bool TryPushToBacklog(Message message, bool onlyIfExists, bool bypassBac return false; } + if (_quiescing) + { + // the backlog this lands in is holding for a handoff (it passed the gate just before the hold rose, or + // is parking behind held work): it sits out the hold like the rest, and gets the same cover + _quiesceDiverted = true; + message.MarkHeldForHandoff(); + } BacklogEnqueue(message); // The correct way to decide to start backlog process is not based on previously empty @@ -1802,6 +1829,13 @@ private void ProcessBridgeBacklog() physical?.Flush(); } + // the held work has reached the replacement: the handoff's protection may stand down now (rule a) + if (_lowerProtectionAfterFlush && !AnyHeldQueued()) + { + _lowerProtectionAfterFlush = false; + _handoffProtection.Clear(); + } + _backlogStatus = BacklogStatus.SettingIdle; physical?.SetIdle(); _backlogStatus = BacklogStatus.Inactive; diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 0f1c256bd..3e48de0df 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -214,7 +214,9 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) /// /// The effective timeout for a particular command: as , and /// for a command that a maintenance handoff diverted into the backlog () no - /// less than the handoff's cover for as long as that stands. Only such a command: an ordinary command on the same + /// less than the handoff's cover, for as long as the command lives: the cover is a property of the held command, + /// not of a window on the server, so it cannot be withdrawn between the replacement establishing and the held + /// work being written to it. Only such a command: an ordinary command on the same /// server, including one issued by an application that has turned relaxation off and runs short timeouts, keeps /// its configured timeout. The sweeps stop at the first command that has not timed out, so a held command at the /// head can delay (never lose) the timeout of an ordinary one behind it, by at most the cover. @@ -222,7 +224,7 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds, Message message) { var result = GetEffectiveTimeoutMilliseconds(configuredMilliseconds); - if (message.HeldForHandoff && _holdCover.IsActive) + if (message.HeldForHandoff) { var floor = Volatile.Read(ref _holdCoverMilliseconds); if (floor > result) result = floor; @@ -230,42 +232,25 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds, Message return result; } - private TickDeadline _holdCover; private int _holdCoverMilliseconds; /// - /// For the duration of a handoff hold plus the reconnect that follows it - including the rest of the window, for - /// which a MoveTo pins reconnection to the named successor - commands against this server time out no sooner - /// than that whole span. Independent of relaxation: a relaxed timeout of zero is the documented way to - /// turn relaxation off, and a sub-second configured timeout with the hold's one-second floor would otherwise - /// time out exactly the work the hold is protecting. + /// The timeout the commands a handoff hold diverts are entitled to: the hold, the rest of the window (for which a + /// MoveTo pins reconnection to the named successor) and one connect attempt, whatever the relaxation setting - a + /// relaxed timeout of zero is the documented way to turn relaxation off, and a sub-second configured timeout with + /// the hold's one-second floor would otherwise time out exactly the work the hold is protecting. Applies to held + /// commands only, for as long as they live; see . + /// Extends, never shortens: a handoff beginning while an earlier one's held work is still queued must not cut + /// that work's cover short. /// private void SetHoldCover(TimeSpan span) { - // extend, never shorten: a handoff beginning while an earlier one's cover is up must not cut that one short var ms = TickDeadline.Clamp(span); - if (_holdCover.IsActive) ms = Math.Max(ms, Volatile.Read(ref _holdCoverMilliseconds)); - Volatile.Write(ref _holdCoverMilliseconds, ms); - _holdCover.TryExtendBy(span, out _); - } - - /// - /// Ends the hold cover early: both connections are fully established again, so the held work is flowing and - /// ordinary timeouts - including a deliberately short one, with relaxation turned off - apply once more. The - /// deadline set by is only the bound for the case where that never happens. - /// - private void ClearHoldCover() => _holdCover.Clear(); - - /// Whether either bridge still has a handoff's protection up; see . - internal bool AnyHandoffPending => interactive?.HandoffPending == true || subscription?.HandoffPending == true; - - /// - /// A bridge's handoff protection has just stood down (its replacement established, or the handoff gave up on it); - /// the server's timeout cover follows the same lifecycle and stands down once no bridge needs it. - /// - internal void OnHandoffProtectionLowered() - { - if (!AnyHandoffPending) ClearHoldCover(); + while (true) + { + var current = Volatile.Read(ref _holdCoverMilliseconds); + if (current >= ms || Interlocked.CompareExchange(ref _holdCoverMilliseconds, ms, current) == current) return; + } } /// diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index b74b81fde..7b977cceb 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1129,10 +1129,6 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // whatever a handoff pointed us at, we are connected now: resume normal resolution ClearHandoffTarget(); - // and a handoff's timeout cover stands down with the last bridge that still needed it: a bridge - // that has not yet retired its connection, or is still reconnecting, keeps it up - OnHandoffProtectionLowered(); - // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; // and if so, is pub/sub sharing it? (if not, subscriptions live on a dedicated connection) diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index 726eb0bbb..d8af405f0 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -501,14 +501,27 @@ protected override void Dispose(bool disposing) */ // written by the test thread, read by the server's own thread(s): store as ticks so the read/write can // be made explicitly visible (a plain TimeSpan field can be missed indefinitely) - public void SetLatency(TimeSpan latency) => Volatile.Write(ref _latencyTicks, latency.Ticks); + public void SetLatency(TimeSpan latency) => SetLatency(latency, only: null); + + /// + /// Hold every reply by , or only the replies to one command: a test that wants the + /// workload slow while a reconnecting client's handshake stays fast (so that a short timeout bites the workload + /// and not the handshake) names the workload's command here. + /// + internal void SetLatency(TimeSpan latency, RedisCommand? only) + { + Volatile.Write(ref _latencyOnly, only is { } command ? (int)command : -1); + Volatile.Write(ref _latencyTicks, latency.Ticks); + } private long _latencyTicks; + private int _latencyOnly = -1; protected override ValueTask ClientPauseAsync(RedisClient client, in RedisRequest request) { var latency = TimeSpan.FromTicks(Volatile.Read(ref _latencyTicks)); - if (latency > TimeSpan.Zero & request.KnownCommand != RedisCommand.QUIT) + var only = Volatile.Read(ref _latencyOnly); + if (latency > TimeSpan.Zero & request.KnownCommand != RedisCommand.QUIT & (only < 0 || (int)request.KnownCommand == only)) { Log($"[{client}] holding {request.Command} response by {latency.TotalMilliseconds}ms"); return new(Task.Delay(latency)); diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index b3fae000e..57f094312 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -854,7 +854,26 @@ public Task MovingUnderSteadyLoadWithRelaxationDisabledStillDrains() // and must keep a floor rather than recycle at once with commands still in flight => SteadyLoadHandoffAsync(configure: c => c.MaintenanceRelaxedTimeout = TimeSpan.Zero); - private async Task SteadyLoadHandoffAsync(Action? configure) + [Fact] + public Task MovingUnderSteadyLoadWithShortTimeoutsAndRelaxationOffKeepsHeldWork() + // The sharp case for the hold's cover: relaxation off, and a timeout shorter than the time a held command spends + // waiting (the drain of the in-flight ones, the replacement's handshake, then its own turn). The held commands + // must keep their cover until they are answered on the replacement - not only until it establishes - or the + // sweep fails them in the gap; everything else keeps the short timeout. Only the workload is slow: the test + // server answers one request at a time, so slowing the handshake too would push *it* past the timeout and + // nothing could ever resume, which is a harness limit and not the behaviour under test. + => SteadyLoadHandoffAsync( + configure: c => + { + c.MaintenanceRelaxedTimeout = TimeSpan.Zero; + c.AsyncTimeout = 1000; + c.SyncTimeout = 1000; + }, + serverLatencyMilliseconds: 300, + latencyOnlyFor: RedisCommand.PING, + workers: 2); // two in flight at most: serially served, their ordinary wait stays inside the timeout + + private async Task SteadyLoadHandoffAsync(Action? configure, int serverLatencyMilliseconds = 40, RedisCommand? latencyOnlyFor = null, int workers = 4) { // The drain used to wait for "no caller work", which under steady traffic never comes: measured live at // about 450 ops/s, the recycle arrived at the end of the 15s window and failed the eight commands in @@ -867,12 +886,12 @@ private async Task SteadyLoadHandoffAsync(Action? configur await using (conn) { // every reply is held briefly, so with several workers something is always in flight - server.SetLatency(TimeSpan.FromMilliseconds(40)); + server.SetLatency(TimeSpan.FromMilliseconds(serverLatencyMilliseconds), latencyOnlyFor); var db = conn.GetDatabase(); using var stop = new CancellationTokenSource(); long completed = 0; var failures = new List(); - var workers = Enumerable.Range(0, 4).Select(_ => Task.Run(async () => + var workerTasks = Enumerable.Range(0, workers).Select(_ => Task.Run(async () => { while (!stop.IsCancellationRequested) { @@ -889,7 +908,8 @@ private async Task SteadyLoadHandoffAsync(Action? configur })).ToArray(); try { - Assert.True(await Poll.UntilAsync(() => Volatile.Read(ref completed) > 20), "the load should be flowing before the handoff"); + // 20 completions can take a while when the workload is deliberately slow and served one at a time + Assert.True(await Poll.UntilAsync(() => Volatile.Read(ref completed) > 20, timeoutMilliseconds: 30_000), "the load should be flowing before the handoff"); var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); @@ -918,7 +938,7 @@ await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutM log.WriteLine($"traffic resumed {resumeWatch.ElapsedMilliseconds}ms after the recycle was observed"); stop.Cancel(); - await Task.WhenAll(workers); + await Task.WhenAll(workerTasks); lock (failures) { log.WriteLine($"completed: {Volatile.Read(ref completed)}; failed: {failures.Count}"); From 5451bf934a9996ff7098c990806b59e9b5b6927f Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 17:59:20 +0000 Subject: [PATCH 18/27] Handoff: one explicit phase per handoff in place of the parallel protection flags Rounds 13 to 17 of review each found a case where two of the handoff's flags disagreed. This replaces them with one object per handoff on each bridge, carrying a phase advanced in one direction by compare-exchange (Holding, AwaitingReplacement, Flushing, Done) and the facts fixed when it began (target, retiring address, whether an address identifies a node), plus one monotonic deadline shared by successive handoffs. Every handoff question - hold new work, flush the backlog, is this failure the handoff's own, is this enqueue covered - is answered from the phase. With that in place: - the hold cover is stamped on each held message as the handoff deadline (an absolute tick), so it cannot be withdrawn after the replacement establishes and does not ratchet across handoffs; the server-level cover and the stand-down-after-flush flag go; - everything entering the backlog during a handoff is held, in BacklogEnqueue alone, which also keeps the timeout sweeps' head-of-line rule sound without skipping; - a connection records that it was made under the successor pin, which is kept until the window expires rather than cleared when the first bridge establishes, so the second bridge's reconnect is not sent back to stale DNS; - a socket address identifies a node only when the handoff has itself seen a second address (a named successor, or DNS moving) and no tunnel is in the way; otherwise only time tells; - classification no longer latches a verdict on ordinary establishment: it is only consulted while a handoff is in progress; - sync waiters re-read their budget in slices only where maintenance notifications can change it; - TickDeadline keeps a passed deadline as the record of when it passed, replacing the hand-rolled relaxed-ended mirror, and owns the single nudge-from-zero helper. --- .../ConnectionMultiplexer.cs | 10 +- .../Maintenance/TickDeadline.cs | 25 +- src/StackExchange.Redis/Message.cs | 32 +- src/StackExchange.Redis/PhysicalBridge.cs | 451 ++++++++++-------- src/StackExchange.Redis/PhysicalConnection.cs | 34 +- .../ServerEndPoint.Maintenance.cs | 142 +++--- src/StackExchange.Redis/ServerEndPoint.cs | 8 +- .../MaintenanceNotificationTests.cs | 2 +- 8 files changed, 379 insertions(+), 325 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index b724361bd..e0a76d333 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2739,12 +2739,16 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u // // Waiting in slices is what makes that re-read reachable: a single wait for the whole budget // that nothing pulses (a genuinely slow command) would time out at the original deadline and - // never see a relaxation, or a handoff's cover, that began after we parked. - const int sliceMilliseconds = 1000; + // never see a relaxation, or a handoff's cover, that began after we parked. Only where the + // budget can change at all, though: with maintenance notifications disabled for the server + // nothing ever relaxes or covers it, and one wait for the whole budget is enough. + var sliceMilliseconds = server is not null && server.MaintenanceMayChangeTimeouts ? 1000 : int.MaxValue; var startedAt = Environment.TickCount; while (true) { - var budget = server?.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds, message) ?? TimeoutMilliseconds; + var budget = server is null + ? TimeoutMilliseconds + : ServerEndPoint.GetEffectiveTimeoutMilliseconds(server.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds), message); var remaining = budget - unchecked(Environment.TickCount - startedAt); if (remaining <= 0) { diff --git a/src/StackExchange.Redis/Maintenance/TickDeadline.cs b/src/StackExchange.Redis/Maintenance/TickDeadline.cs index b9842412d..7f344b3f0 100644 --- a/src/StackExchange.Redis/Maintenance/TickDeadline.cs +++ b/src/StackExchange.Redis/Maintenance/TickDeadline.cs @@ -6,10 +6,13 @@ namespace StackExchange.Redis.Maintenance; /// /// A deadline expressed in terms, where zero means "unset". One place for the /// three invariants every such field needs: a genuine zero tick count is nudged to one so it cannot read as unset; -/// an observed expiry is cleared (by compare-exchange, so a concurrent extension is not undone), so that a tick count -/// wrapping ~24.8 days later cannot bring a stale deadline back to life; and extension is monotonic, never shortening -/// a deadline that is already later. Used for the relaxed-timeout window, the handoff hold cover and the per-bridge -/// handoff protection, which all had their own copies of this before. +/// a long-passed deadline is cleared (by compare-exchange, so a concurrent extension is not undone), so that a tick +/// count wrapping ~24.8 days later cannot bring it back to life; and extension is monotonic, never shortening a +/// deadline that is already later. A deadline that has merely passed is kept, readable through , +/// so it also records when it passed: fault attribution asks how recently the relaxed window closed, and a +/// held command's cover is stamped from the handoff deadline whether or not the handoff is still running. Used for +/// the relaxed-timeout window, the successor pin and the per-bridge handoff deadline, which all had their own copies +/// of this before. /// internal struct TickDeadline { @@ -23,7 +26,7 @@ internal struct TickDeadline /// private const int MaxMilliseconds = int.MaxValue / 4; - /// The raw deadline, or zero when unset. For callers that record *when* a window was due to end. + /// The raw deadline, passed or not, or zero when unset (or so long passed that it has been cleared). public int Ticks => Volatile.Read(ref _ticks); public void SetIn(TimeSpan span) => Volatile.Write(ref _ticks, NudgeFromZero(unchecked(Environment.TickCount + Clamp(span)))); @@ -63,7 +66,10 @@ public bool TryExtendBy(TimeSpan span, out int deadline) } } - /// Milliseconds until the deadline, or zero if unset or passed; a passed deadline is cleared. + /// + /// Milliseconds until the deadline, or zero if unset or passed. A passed deadline is kept (see ) + /// until it is so long past that a wrapping tick count could make it look future again, and then cleared. + /// public int RemainingMilliseconds() { var deadline = Volatile.Read(ref _ticks); @@ -71,12 +77,13 @@ public int RemainingMilliseconds() var remaining = unchecked(deadline - Environment.TickCount); if (remaining > 0) return remaining; - // expired: clear it, but only if nobody has moved it on in the meantime - Interlocked.CompareExchange(ref _ticks, 0, deadline); + // long passed: clear it, but only if nobody has moved it on in the meantime + if (remaining < -MaxMilliseconds) Interlocked.CompareExchange(ref _ticks, 0, deadline); return 0; } public bool IsActive => RemainingMilliseconds() > 0; - private static int NudgeFromZero(int ticks) => ticks == 0 ? 1 : ticks; + /// Zero means "unset" (or "never"), so a tick count that lands on it moves by one. + public static int NudgeFromZero(int ticks) => ticks == 0 ? 1 : ticks; } diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index cc0135f1c..0177e14f1 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -906,16 +906,34 @@ internal void SetRequeued() } /// - /// Whether a maintenance handoff diverted this message off a live connection into the backlog (or requeued it - /// after retiring the connection it was headed for). Such a message is waiting on the client's own choice, not - /// on the server, so the handoff's timeout cover applies to it - and only to it: an ordinary command on the same - /// server keeps its configured timeout. + /// Whether this message entered a backlog during a maintenance handoff - diverted off a live connection by the + /// hold, requeued after the connection it was headed for was retired, or queued while the bridge reconnected to + /// the replacement. Such a message is waiting on the client's own choice, not on the server, so the handoff's + /// timeout cover applies to it (and only to it: an ordinary command on the same server keeps its configured + /// timeout) for as long as it lives, including after it has been written to the replacement. /// - internal bool HeldForHandoff => _heldForHandoff; + internal bool HeldForHandoff => Volatile.Read(ref _holdCoverDeadlineTicks) != 0; - internal void MarkHeldForHandoff() => _heldForHandoff = true; + /// + /// The handoff's deadline, in terms, until which this message cannot time + /// out: the cover is a property of the message, stamped when it is held, so nothing that happens to the handoff + /// afterwards can withdraw it. Zero if the message was never held. + /// + internal int HoldCoverDeadlineTicks => Volatile.Read(ref _holdCoverDeadlineTicks); + + /// Holds the message for a handoff, covered until ; a later deadline replaces an earlier one, never the reverse. + internal void MarkHeldForHandoff(int deadlineTicks) + { + if (deadlineTicks == 0) return; // no deadline to cover it until: a hold that was over before the enqueue completed + while (true) + { + var current = Volatile.Read(ref _holdCoverDeadlineTicks); + if (current != 0 && unchecked(deadlineTicks - current) <= 0) return; + if (Interlocked.CompareExchange(ref _holdCoverDeadlineTicks, deadlineTicks, current) == current) return; + } + } - private volatile bool _heldForHandoff; + private int _holdCoverDeadlineTicks; private int _writeTickCount; diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 4be595fae..95fda0916 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -133,24 +133,112 @@ public enum State : byte /// internal void MarkNeedsReconnect() => Volatile.Write(ref _needsReconnect, true); - private volatile bool _quiescing, _quiesceRecycled, _quiesceSuccessorPinned, _quiesceDiverted, _quiesceEndpointIdentityMeaningful, _lowerProtectionAfterFlush; - private PhysicalConnection? _quiesceTarget; - private System.Net.IPEndPoint? _quiesceRetiringEndPoint; + /// + /// Where a maintenance handoff stands on this bridge. One variable, advanced in one direction, and the only + /// thing every handoff question is answered from: whether new work is held, whether the backlog may flush, + /// whether a failure is the handoff's own, and whether work enqueued now is covered. (Parallel flags for the + /// same facts kept disagreeing with each other in the races between them.) + /// + private enum HandoffPhase + { + /// + /// The connection current when the hold began (the target) is being drained: new caller work is held in + /// the backlog, the backlog does not flush, and the drain is waiting for the target's in-flight work. + /// + Holding, + + /// + /// The target is gone - retired by the drain, or it went away by itself - and whatever is current is not + /// yet known to be a bona fide replacement. New work is still held and the backlog still does not flush, + /// because writing to a connection the server is about to close would only hand it work to fail. + /// + AwaitingReplacement, + + /// + /// A bona fide replacement is current and established, and the held work is being written to it. The + /// handoff's protection stands until the last held message has been written: a failure before that would + /// still abandon them under a FailFast policy. + /// + Flushing, + + /// Nothing left to protect; or the deadline passed; or the bridge was disposed. + Done, + } /// - /// While active, a maintenance handoff is replacing this bridge's connection, and work the hold diverted into - /// the backlog survives the reconnect under a FailFast policy as well as the hold itself: a failure on the way - /// to the replacement is not a reason to abandon it. Raised (monotonically: a later hold may extend it, never - /// shorten it, since an earlier hold's diverted work may still be queued) in ; - /// lowered when a bona fide replacement is fully established, when the handoff finds it has nothing to protect, - /// or by the deadline. The server's timeout cover shares exactly this lifecycle, across both bridges. + /// One maintenance handoff on this bridge: its phase, and the facts fixed when it began. A new object per + /// handoff, so that a transition requested against an earlier handoff (a backlog pass finishing an earlier + /// flush, say) cannot land on a later one. /// - private Maintenance.TickDeadline _handoffProtection; + private sealed class Handoff + { + private int _phase; // HandoffPhase + + /// The connection the hold began on, or a reconnect adopted in its place; null if the bridge had none. + public volatile PhysicalConnection? Target; + + /// Whether this handoff retired a connection (rather than finding it gone). + public volatile bool Retired; + + /// The address and port of the node being retired, as seen from the connection that was told; null if unreadable. + public readonly System.Net.IPEndPoint? RetiringEndPoint; - /// The handoff has nothing left to protect on this bridge: its protection stands down. - internal void ClearHandoffPending() => _handoffProtection.Clear(); + /// + /// Whether a socket's remote address says which node it reached. Only when the handoff has itself seen a + /// second address - the server named a successor, or DNS moved to a new address - and there is no tunnel + /// in the way: then a reconnect reporting the retiring address is back on the retiring node. Behind a + /// tunnel, or when the only address ever involved is the one in front of the cluster, every connection + /// reports the same address and comparing it would brand the genuine replacement doomed forever; then + /// only time tells. + /// + public readonly bool AddressIdentifiesNode; + + public Handoff(PhysicalConnection? target, System.Net.IPEndPoint? retiringEndPoint, bool addressIdentifiesNode) + { + Target = target; + RetiringEndPoint = retiringEndPoint; + AddressIdentifiesNode = addressIdentifiesNode; + } - internal bool HandoffPending => _handoffProtection.IsActive; + public HandoffPhase Phase => (HandoffPhase)Volatile.Read(ref _phase); + + /// Advances the phase, if it is still where the caller saw it. + public bool TryAdvance(HandoffPhase from, HandoffPhase to) + => Interlocked.CompareExchange(ref _phase, (int)to, (int)from) == (int)from; + + /// Ends the handoff whatever its phase: the deadline passed, or the bridge was disposed. + public void Finish() => Volatile.Write(ref _phase, (int)HandoffPhase.Done); + } + + private volatile Handoff? _handoff; + + /// + /// Until when a handoff's protection can stand at the latest: the hold, the rest of the window and one connect + /// attempt. Shared by successive handoffs and only ever extended, since an earlier handoff's held work may + /// still be queued when a later one begins. Its ticks also stamp each held message's timeout cover, so it is + /// left to lapse rather than cleared when a handoff finishes early: the cover is the message's, for its whole + /// life, and must not be withdrawn between the replacement establishing and the message being answered. + /// + private Maintenance.TickDeadline _handoffDeadline; + + /// The handoff in progress, if there is one and its deadline has not passed (a passed deadline ends it). + private bool TryGetHandoff([NotNullWhen(true)] out Handoff? handoff) + { + handoff = _handoff; + if (handoff is null || handoff.Phase == HandoffPhase.Done) return false; + if (_handoffDeadline.IsActive) return true; + handoff.Finish(); + return false; + } + + private HandoffPhase Phase => TryGetHandoff(out var handoff) ? handoff.Phase : HandoffPhase.Done; + + /// + /// A handoff's hold is in force: whatever is current is the target, or is not known to be a bona fide + /// replacement. New work is queued and the backlog does not flush; the moment a replacement is current and + /// established (or the deadline passes), the hold is over and the backlog flushes to it. + /// + private bool HoldUnresolved => Phase is HandoffPhase.Holding or HandoffPhase.AwaitingReplacement; /// /// The last resort when the write lock could not be had for the whole of the drain's deadline: retires the @@ -160,26 +248,28 @@ public enum State : byte /// reconnect. Before handoffs existed the recycle was always lockless, so this is the old behaviour kept as /// the fallback rather than the rule. /// - internal bool RetireQuiesceTargetWithoutLock(string reason) + internal bool RetireTargetWithoutLock(string reason) { - var target = _quiesceTarget; - if (target is null || _quiesceRecycled || !ReferenceEquals(physical, target)) return false; + if (!TryGetHandoff(out var handoff) || handoff.Target is not { } target || !ReferenceEquals(physical, target)) return false; target.MarkRetiredForHandoff(); - _quiesceRecycled = true; + handoff.Retired = true; + handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); RetireConnection(target, reason + " (without the write lock: last resort)"); return true; } /// - /// The rules the handoff's protection state machine keeps, stated once so that the next edit is checked - /// against them rather than against the last edit: + /// The rules the handoff keeps, stated once so that the next edit is checked against them rather than + /// against the last edit: /// /// (a) protection stands down only when the held work is safe - written to a bona fide - /// replacement, or carrying its own cover - never merely because a replacement exists; + /// replacement, or past the deadline that also bounds its cover - never merely because a replacement + /// exists; /// (b) classifying a connection has three outcomes, replacement, doomed or unknown, and unknown always /// defers to a later pass - it is never acted on as either of the others; - /// (c) every trust snapshot taken at - the pin, the retiring endpoint, - /// whether endpoint identity is meaningful - is validated where it is used, not assumed from capture. + /// (c) what a connection is - the target, made under the successor pin, established - is + /// recorded on the connection when it happens and read from there; the handoff's phase says only where the + /// flow stands, and nothing about a connection is inferred from it. /// /// private enum Replacement @@ -195,48 +285,33 @@ private enum Replacement } /// - /// Classifies a connection against the handoff in progress. Target status dominates, including over an earlier - /// handoff's latch. A successor pin is trusted only while it is still in force (rule c): the first bridge's - /// replacement establishing clears the pin, after which the other bridge's retry resolves the endpoint again - /// and may land anywhere. Where endpoint identity is meaningless (a tunnel, a fixed front address) only time - /// tells: once the retire has resolved or the drain has let go, nothing else will arrive, so what is current is - /// the replacement; until then, unknown. Otherwise the landed-on endpoint decides, before and after the - /// retire, with an unreadable address meaning unknown. A verdict of replacement is latched on an established - /// connection so the hot path does not re-derive it. + /// Classifies a connection against a handoff. The connection's own facts come first (rule c): the target, or a + /// reconnect adopted as one, is doomed whatever else is true of it; a connection made under the successor pin + /// went where the server said, which is not the node being retired. Then the address: where the remote address + /// identifies the node, landing back on the retiring address is doomed and anywhere else is the replacement, + /// with an unreadable address meaning unknown. Where it identifies nothing (a tunnel, a fixed front address) + /// only time tells: while the target is still being drained a different current connection cannot be judged; + /// once the target is gone, nothing else will arrive, so what is current is the replacement. /// - private Replacement Classify(PhysicalConnection connection) + private static Replacement Classify(Handoff handoff, PhysicalConnection connection) { if (connection.WasHandoffTarget) return Replacement.Doomed; - if (connection.KnownReplacement) return Replacement.Yes; - Replacement verdict; - if (_quiesceSuccessorPinned && ServerEndPoint.HandoffTarget is not null) + if (connection.ConnectedViaHandoffTarget) return Replacement.Yes; + if (!handoff.AddressIdentifiesNode) { - verdict = Replacement.Yes; + return handoff.Phase == HandoffPhase.Holding ? Replacement.Unknown : Replacement.Yes; } - else if (!_quiesceEndpointIdentityMeaningful) + var retiring = handoff.RetiringEndPoint; + if (retiring is null) { - verdict = _quiesceRecycled || !_quiescing ? Replacement.Yes : Replacement.Unknown; - } - else - { - var retiring = _quiesceRetiringEndPoint; - if (retiring is null) - { - // nothing to compare against: the connection that was told had no readable address. Deferring would - // hold traffic until the protection expired with nothing ever able to classify, so the replacement - // is taken at its word - verdict = Replacement.Yes; - } - else - { - var landedOn = connection.TryGetRemoteEndPoint(); - if (landedOn is null) verdict = Replacement.Unknown; - else if (landedOn.Equals(retiring)) verdict = Replacement.Doomed; - else verdict = Replacement.Yes; - } + // nothing to compare against: the connection that was told had no readable address. Deferring would + // hold traffic until the deadline with nothing ever able to classify, so the replacement is taken at + // its word + return Replacement.Yes; } - if (verdict == Replacement.Yes && connection.EverFullyEstablished) connection.MarkKnownReplacement(); - return verdict; + var landedOn = connection.TryGetRemoteEndPoint(); + if (landedOn is null) return Replacement.Unknown; + return landedOn.Equals(retiring) ? Replacement.Doomed : Replacement.Yes; } /// Whether the backlog still holds work a handoff diverted; the protection stands until it has been written (rule a). @@ -250,15 +325,15 @@ private bool AnyHeldQueued() } /// - /// Whether a connection failure belongs to a maintenance handoff in progress: while the protection is up, any - /// failure of a connection that is not a known, established replacement is the handoff's own - the target - /// (retired by the drain, closed by the server at the end of the window, or dropped by itself), a reconnect - /// that landed back on the node being retired, or an attempt to bring up the replacement that never completed - /// its handshake, whatever the failure type. Only a replacement that was fully established and then failed is - /// an ordinary failure, and a FailFast policy applies to it as usual, even while the protection is still up. + /// Whether a connection failure belongs to a maintenance handoff in progress: the target (retired by the + /// drain, closed by the server at the end of the window, or dropped by itself), a reconnect that landed back + /// on the node being retired, or an attempt to bring up the replacement that never completed its handshake, + /// whatever the failure type. Only a replacement that was fully established and then failed is an ordinary + /// failure, and a FailFast policy applies to it as usual, even while the handoff is still flushing. /// private bool IsHandoffOwnFailure(PhysicalConnection connection) - => HandoffPending && !connection.KnownReplacement; + => TryGetHandoff(out var handoff) + && (!connection.EverFullyEstablished || Classify(handoff, connection) != Replacement.Yes); /// /// Starts holding caller work back from the current connection while a maintenance handoff lets what has @@ -272,71 +347,81 @@ private bool IsHandoffOwnFailure(PhysicalConnection connection) /// something in flight, so the recycle waited for the end of the announced window and then failed whatever /// was in flight at that instant. /// - /// For how long at most the handoff's protection stands - the FailFast shield here and - /// the timeout cover on the server - if no bona fide replacement establishes first. + /// For how long at most the handoff stands - the FailFast shield here and the timeout + /// cover on the held commands - if no bona fide replacement establishes and flushes first. /// The address and port of the node being retired, as seen from the connection /// that was told, if known; see . - /// Whether reconnection is pinned to a named successor, in which case any - /// connection other than the target is a replacement. - /// Whether a socket's remote endpoint identifies the node it reached; - /// behind a tunnel or a fixed front address it does not, and only time tells a replacement apart. - internal void BeginQuiesce(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, bool successorPinned, bool endpointIdentityMeaningful) + /// Whether a socket's remote address identifies the node it reached; see + /// . + internal void BeginHold(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, bool addressIdentifiesNode) { var target = physical; target?.MarkHandoffTarget(); - _quiesceTarget = target; - _quiesceRetiringEndPoint = retiringEndPoint; - _quiesceSuccessorPinned = successorPinned; - _quiesceEndpointIdentityMeaningful = endpointIdentityMeaningful; - _quiesceRecycled = false; - // a hold that begins while an earlier handoff's protection is still up (the bridge still reconnecting from - // it, with that handoff's diverted work queued) extends it rather than replacing it, and remembers that - // there is older work to protect even if this hold itself diverts nothing - var inheritedProtection = _handoffProtection.IsActive; - _quiesceDiverted = inheritedProtection; // older work still queued counts as diverted work to protect - _lowerProtectionAfterFlush = false; - _handoffProtection.TryExtendBy(protection, out _); - _quiescing = true; - } - - internal void EndQuiesce() - { - if (!_quiescing) return; - _quiescing = false; - _quiesceTarget = null; - // Store the flag, fence, then load the queue - the mirror image of WakeIfHoldReleased, which every held - // enqueue (single, batch, requeue) goes through and which is the only enqueue-side half of this pairing. - // StartBacklogProcessor wakes a processor that is already running. - Interlocked.MemoryBarrier(); - if (BacklogHasItems) StartBacklogProcessor(); + // the deadline before the phase: a reader that sees the new handoff must also see a live deadline, or it + // would end it on the spot. Extended, never reset: an earlier handoff's held work may still be queued. + _handoffDeadline.TryExtendBy(protection, out _); + _handoff = new Handoff(target, retiringEndPoint, addressIdentifiesNode); } /// - /// Whether a message should be held in the backlog for the replacement connection rather than written: only - /// while a handoff is draining a connection that is still live (once it has dropped, the ordinary disconnected - /// rules and the backlog policy apply), and only for a caller's work. Internal traffic (keep-alives, probes) - /// still goes to the old connection, and so does QUIT, which is never queued. + /// The drain loop has ended, whatever it achieved. A target it could not retire (the lock never came) is left + /// to the server's close, and the hold stays until a replacement is current; a replacement that established + /// while the drain was still judging the target is judged again now that the target is gone. /// - private bool ShouldHoldForHandoff(Message message) => HoldActive && IsHoldable(message); + internal void EndDrain() + { + if (!TryGetHandoff(out var handoff)) return; + handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); + ResolveIfReplacementCurrent(handoff); + } + + /// Resolves the hold if the current connection is an established bona fide replacement. + private void ResolveIfReplacementCurrent(Handoff handoff) + { + if (physical is { EverFullyEstablished: true } current && Classify(handoff, current) == Replacement.Yes) + { + OnReplacementEstablished(handoff); + } + } /// - /// A handoff's hold is in force: whatever is current is not known to be a bona fide replacement - it is the - /// target still, a reconnect that landed back on the node being retired, or one not classifiable yet - and - /// the server is going to close it, or might. While it holds, new work is queued and the backlog does not - /// flush, because writing to that connection would only hand the server work to fail. The moment a bona fide - /// replacement is current (or the protection has expired), the hold is over and the backlog flushes to it. + /// A bona fide replacement is current and established: the hold is over and the backlog flushes to it. The + /// handoff stands until the held work has been written (rule a); if nothing is held, it is done now. /// - private bool HoldUnresolved + private void OnReplacementEstablished(Handoff handoff) { - get + if (!(handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.Flushing) + || handoff.TryAdvance(HandoffPhase.AwaitingReplacement, HandoffPhase.Flushing))) { - if (!((_quiescing && !_quiesceRecycled) || HandoffPending)) return false; - var current = physical; - if (current is null || ReferenceEquals(current, _quiesceTarget)) return true; - return Classify(current) != Replacement.Yes; + return; // already flushing, or done } + TryFinishFlush(handoff); + // Store the phase, fence, then load the queue - the mirror image of WakeIfHoldReleased, which every held + // enqueue (single, batch, requeue) goes through. With a full fence on both sides at least one of them + // sees the other's store, so a message held concurrently with the release is not stranded; neither side + // starts a processor for an empty backlog, and StartBacklogProcessor wakes one that is already running. + Interlocked.MemoryBarrier(); + if (BacklogHasItems) StartBacklogProcessor(); + } + + /// + /// The held work has all been written: nothing left to protect (rule a). A message held between the check and + /// the transition (a requeue by a writer that reached the lock late) keeps its cover - that is stamped on the + /// message - and is flushed by the processor like any other; only the FailFast shield is down for it. + /// + private void TryFinishFlush(Handoff handoff) + { + if (handoff.Phase == HandoffPhase.Flushing && !AnyHeldQueued()) handoff.TryAdvance(HandoffPhase.Flushing, HandoffPhase.Done); } + /// + /// Whether a message should be held in the backlog for the replacement connection rather than written: only + /// while a handoff is draining a connection that is still live (once it has dropped, the ordinary disconnected + /// rules and the backlog policy apply), and only for a caller's work. Internal traffic (keep-alives, probes) + /// still goes to the old connection, and so does QUIT, which is never queued. + /// + private bool ShouldHoldForHandoff(Message message) => HoldActive && IsHoldable(message); + /// A handoff's hold is unresolved, and the current connection is live. private bool HoldActive => HoldUnresolved && IsConnected && !NeedsReconnect; @@ -372,18 +457,18 @@ private static bool ShouldHoldBatch(List messages) /// Why; for the trace. /// Bounds the wait for the write lock (together with the mutex's own timeout, whichever /// ends first); the drain re-waits until this deadline, so it is the deadline that decides how long overall. - /// null if the connection could not be retired yet - caller work in flight, or the lock not - /// obtained inside the deadline - so the drain should try again; otherwise whether this call, or an earlier - /// one, replaced it. - internal async ValueTask TryRetireQuiesceTargetAsync(bool force, string reason, CancellationToken lockDeadline) + /// null if the connection could not be retired yet - caller work in flight, the lock not + /// obtained inside the deadline, or the current connection not classifiable yet - so the drain should try + /// again; otherwise whether this handoff retired a connection. + internal async ValueTask TryRetireTargetAsync(bool force, string reason, CancellationToken lockDeadline) { - var target = _quiesceTarget; - if (_quiesceRecycled) return true; + if (!TryGetHandoff(out var handoff)) return false; + if (handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return handoff.Retired; // Lock-free first: the drain polls this every 20ms, and the in-flight question only needs the connection's // own lock. The write lock is taken only for a pass that is likely to retire, where it has to be held // across the check and the marking. - if (!force && target is not null && ReferenceEquals(physical, target) && target.HasCallerMessagesAwaitingResponse()) return null; + if (!force && handoff.Target is { } busy && ReferenceEquals(physical, busy) && busy.HasCallerMessagesAwaitingResponse()) return null; PhysicalConnection toRetire; bool gotLock; @@ -397,46 +482,38 @@ private static bool ShouldHoldBatch(List messages) } catch (ObjectDisposedException) { - // the bridge was disposed mid-handoff: nothing left to retire, and no replacement is coming that the - // protection could be waiting for - ClearHandoffPending(); + // the bridge was disposed mid-handoff: nothing left to retire, and no replacement is coming + handoff.Finish(); return false; } if (!gotLock) return null; try { - if (_quiesceRecycled) return true; + if (handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return handoff.Retired; var current = physical; - if (target is not null && ReferenceEquals(current, target)) + if (handoff.Target is { } target && ReferenceEquals(current, target)) { toRetire = target; } else { // The connection the hold began on is gone - it went away by itself (the server closed it, or it - // faulted), or there was none when the hold began because it had already dropped. + // faulted), or there was none when the hold began because the bridge was already disconnected. + // The phase says so first, so that whatever is current is judged as of a target that is gone. + handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); if (current is null) { - // Nothing to retire. Whatever the hold diverted stays protected for the reconnect that is - // coming; only a hold that diverted nothing (and inherited no earlier handoff's work) has - // nothing to protect. - _quiesceRecycled = true; - if (!_quiesceDiverted) ClearHandoffPending(); + // reconnecting: the reconnect is judged when it establishes, and whatever the hold diverted + // stays protected for it until then return false; } - switch (Classify(current)) + switch (Classify(handoff, current)) { case Replacement.Yes: // the reconnect is the replacement (a pinned successor, or demonstrably another node): - // retiring it would only force a second connect cycle. If it is already up, the - // protection stands down once the held work has reached it; if it is still connecting, - // its establishment will see to that. - _quiesceRecycled = true; - if (current.EverFullyEstablished) - { - if (AnyHeldQueued()) _lowerProtectionAfterFlush = true; - else ClearHandoffPending(); - } + // retiring it would only force a second connect cycle. If it is already up, the hold is + // over now; if it is still connecting, its establishment will see to that. + if (current.EverFullyEstablished) OnReplacementEstablished(handoff); return false; case Replacement.Unknown: // not classifiable yet - a socket still connecting, say (rule b): never retired on a guess, @@ -444,9 +521,9 @@ private static bool ShouldHoldBatch(List messages) return null; default: // back on the node being retired: what is current now is what the handoff replaces - toRetire = current; current.MarkHandoffTarget(); - _quiesceTarget = current; + handoff.Target = current; + toRetire = current; break; } } @@ -456,7 +533,8 @@ private static bool ShouldHoldBatch(List messages) // TryRequeueForHandoff), and the backlog processor skips it too, so the teardown itself can happen // after the release rather than stalling every writer behind event handlers and socket shutdown. toRetire.MarkRetiredForHandoff(); - _quiesceRecycled = true; + handoff.Retired = true; + handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); } finally { @@ -508,6 +586,7 @@ private void RetireConnection(PhysicalConnection connection, string reason) public void Dispose() { isDisposed = true; + _handoff?.Finish(); // nothing left to protect, and no replacement is coming // If there's anything in the backlog and we're being torn down - exfil it immediately (e.g. so all awaitables complete) AbandonPendingBacklog(new ObjectDisposedException("Connection is being disposed"), null); try @@ -861,9 +940,10 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur } Trace($"OnConnectionFailed: {connection}"); // If we're configured to, fail all pending backlogged messages - but not for a maintenance handoff, which - // is a replacement we chose, and not while a handoff is holding work back: that backlog is work diverted - // off a live connection, which would have been written and answered, so a death of the old connection - // during the hold (the window expiring, a blip) must leave it for the replacement to flush + // is a replacement we chose, and not for a failure that is a handoff's own: the backlog then holds work + // diverted off a live connection, which would have been written and answered, so a death of the old + // connection (the window expiring, a blip) or a failed attempt to reach the replacement must leave it for + // the replacement to flush if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true && failureType != ConnectionFailureType.MaintenanceHandoff && !IsHandoffOwnFailure(connection)) { @@ -967,16 +1047,10 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) { reportNextFailure = reconfigureNextFailure = true; LastException = null; - // only a bona fide replacement establishing lowers the protection for this bridge - not the very - // connection the hold is draining completing a handshake it was in the middle of when the hold began, - // and not a reconnect that landed back on the node being retired - and even then only once the work - // the hold diverted has been written to it (rule a): until the backlog processor has flushed the held - // messages, a failure would still abandon them under a FailFast policy - if (Classify(connection) == Replacement.Yes) - { - if (AnyHeldQueued()) _lowerProtectionAfterFlush = true; - else _handoffProtection.Clear(); - } + // only a bona fide replacement establishing ends the hold for this bridge - not the very connection + // the hold is draining completing a handshake it was in the middle of when the hold began, and not a + // reconnect that landed back on the node being retired (rule c: the connection's own facts decide) + if (TryGetHandoff(out var handoff) && Classify(handoff, connection) == Replacement.Yes) OnReplacementEstablished(handoff); Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -1393,13 +1467,6 @@ private bool TryPushToBacklog(Message message, bool onlyIfExists, bool bypassBac return false; } - if (_quiescing) - { - // the backlog this lands in is holding for a handoff (it passed the gate just before the hold rose, or - // is parking behind held work): it sits out the hold like the rest, and gets the same cover - _quiesceDiverted = true; - message.MarkHeldForHandoff(); - } BacklogEnqueue(message); // The correct way to decide to start backlog process is not based on previously empty @@ -1419,9 +1486,23 @@ internal void AcceptRerouted(Message message) StartBacklogProcessor(); } + /// + /// The one place a message enters the backlog, and so the one place the hold's cover is applied: whatever + /// enters during a handoff is held - diverted off the live connection, queued while the bridge reconnects to + /// the replacement, or parked behind held work being flushed - and gets the same cover, to the same deadline. + /// The shared deadline is also what keeps the timeout sweeps' head-of-line rule sound: a message enqueued + /// later never times out before one enqueued earlier, because an earlier held message expires at the same + /// deadline or sooner, and an ordinary message can only be enqueued after the handoff is done, which is when + /// nothing held is left or that deadline has passed. + /// + /// The message to queue. + /// The message was diverted by a handoff's hold (or requeued after the retire): held + /// whatever the phase is by now, since a hold that resolved between the gate and the enqueue does not make it + /// ordinary work. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void BacklogEnqueue(Message message) + private void BacklogEnqueue(Message message, bool diverted = false) { + if (diverted || Phase != HandoffPhase.Done) message.MarkHeldForHandoff(_handoffDeadline.Ticks); bool wasEmpty = !BacklogHasItems; // important that this *precedes* enqueue, to play well with HasPendingCallerFacingItems Interlocked.Increment(ref _backlogCurrentEnqueued); @@ -1464,10 +1545,10 @@ private bool AnyCallerFacingQueued() /// /// Queues work held for a maintenance handoff - diverted at the gate, or requeued by a writer that reached the /// lock to find its connection retired - and, through , makes sure it is not - /// stranded if the hold is released concurrently: the queue is stored, then a full fence, then the flag is - /// loaded - the mirror image of , which stores the flag, fences, then loads the queue. - /// With a full fence on both sides at least one of them sees the other's store, so one side or the other starts - /// the processor; neither starts one for an empty backlog. + /// stranded if the hold is released concurrently: the queue is stored, then a full fence, then the phase is + /// loaded - the mirror image of , which stores the phase, fences, then + /// loads the queue. With a full fence on both sides at least one of them sees the other's store, so one side or + /// the other starts the processor; neither starts one for an empty backlog. /// /// /// A requeued message is appended, not inserted at the head: a head insert would buy only a best effort at the @@ -1480,37 +1561,31 @@ private bool AnyCallerFacingQueued() /// it keeps its write time, since the budget it has spent still counts, and only forgets that connection. private void EnqueueHeldAndWakeIfReleased(Message message, bool requeue = false) { - // before the enqueue, not after: a drain pass deciding whether there is anything to protect must not see - // the message in the backlog and the flag still off (it is monotonic within a hold, so early is harmless) - _quiesceDiverted = true; - message.MarkHeldForHandoff(); if (requeue) message.SetRequeued(); else message.SetEnqueued(null); - BacklogEnqueue(message); + BacklogEnqueue(message, diverted: true); WakeIfHoldReleased(); } /// A batch held as a unit: every member queued, then one fence and one check for the batch. private void EnqueueHeldBatchAndWakeIfReleased(List messages) { - _quiesceDiverted = true; foreach (var message in messages) { - message.MarkHeldForHandoff(); message.SetEnqueued(null); - BacklogEnqueue(message); + BacklogEnqueue(message, diverted: true); } WakeIfHoldReleased(); } /// - /// The enqueue side of the pairing with , and the only one: the queue has been stored - /// (by whichever helper called this), then a full fence here, then the flag is loaded. + /// The enqueue side of the pairing with , and the only one: the queue + /// has been stored (by whichever helper called this), then a full fence here, then the phase is loaded. /// private void WakeIfHoldReleased() { Interlocked.MemoryBarrier(); - if (!_quiescing) StartBacklogProcessor(); + if (!HoldUnresolved) StartBacklogProcessor(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -1593,21 +1668,21 @@ private void CheckBacklogForTimeouts() // Because peeking at the backlog, checking message and then dequeuing, is not thread-safe, we do have to use // a lock here, for mutual exclusion of backlog DEQUEUERS. Unfortunately. // But we reduce contention by only locking if we see something that looks timed out. - // a command a maintenance handoff diverted here is covered by the handoff for as long as that stands; an - // ordinary command keeps the timeout above (a held one at the head delays, never loses, the one behind it) - int TimeoutFor(Message m) => m.HeldForHandoff ? ServerEndPoint.GetEffectiveTimeoutMilliseconds(Multiplexer.TimeoutMilliseconds, m) : timeout; + // a command a maintenance handoff held is covered until the handoff's deadline; an ordinary command keeps + // the timeout above. The head-of-line rule below still holds across the two: see BacklogEnqueue + static int TimeoutFor(Message m, int timeout) => m.HeldForHandoff ? ServerEndPoint.GetEffectiveTimeoutMilliseconds(timeout, m) : timeout; while (_backlog.TryPeek(out Message? message)) { // See if the message has pass our async timeout threshold // Note: All timed out messages must be dequeued, even when no completion is needed, to be able to dequeue and complete other timed out messages. - if (!message.HasTimedOut(now, TimeoutFor(message), out var _)) break; // not a timeout - we can stop looking + if (!message.HasTimedOut(now, TimeoutFor(message, timeout), out var _)) break; // not a timeout - we can stop looking lock (_backlog) { // Peek again since we didn't have lock before... // and rerun the exact same checks as above, note that it may be a different message now if (!_backlog.TryPeek(out message)) break; - if (!message.HasTimedOut(now, TimeoutFor(message), out var _)) break; + if (!message.HasTimedOut(now, TimeoutFor(message, timeout), out var _)) break; if (!BacklogTryDequeue(out var message2) || (message != message2)) // consume it for real { @@ -1829,12 +1904,8 @@ private void ProcessBridgeBacklog() physical?.Flush(); } - // the held work has reached the replacement: the handoff's protection may stand down now (rule a) - if (_lowerProtectionAfterFlush && !AnyHeldQueued()) - { - _lowerProtectionAfterFlush = false; - _handoffProtection.Clear(); - } + // the held work has reached the replacement: nothing left to protect (rule a) + if (_handoff is { } handoff) TryFinishFlush(handoff); _backlogStatus = BacklogStatus.SettingIdle; physical?.SetIdle(); diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 5052360fe..ef9d76cd1 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -316,6 +316,7 @@ internal async Task BeginConnectAsync(ILogger? log) { Trace($"handoff: connecting to {Format.ToString(handoffTarget)} in place of {Format.ToString(endpoint)}"); connectTo = handoffTarget; + _connectedViaHandoffTarget = true; } if (tunnel is not null) { @@ -988,18 +989,7 @@ internal void GetStormLog(StringBuilder sb) // volatile: each is written on one thread and read on others as part of the handoff's identity test, in the // very races that test exists for; a plain field could be read stale exactly then - private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished, _knownReplacement; - - /// - /// Latched once this connection, fully established, has been found to be a bona fide replacement for one a - /// maintenance handoff retired (see PhysicalBridge.IsBonaFideReplacement): the answer is stable once - /// true, and the test behind it reads the socket's remote address, which is not something to repeat on every - /// write. Cleared if the connection later becomes a handoff's target itself. A failure of a connection that is - /// *not* a known replacement, while a handoff's protection is up, is the handoff's own. - /// - internal bool KnownReplacement => _knownReplacement; - - internal void MarkKnownReplacement() => _knownReplacement = true; + private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished, _connectedViaHandoffTarget; /// /// Whether this connection was ever the one a maintenance handoff held work back from. A durable fact about the @@ -1009,17 +999,19 @@ internal void GetStormLog(StringBuilder sb) /// internal bool WasHandoffTarget => _wasHandoffTarget; - /// Target status dominates: a connection that was a replacement for an earlier handoff and is now the one being retired is a target, not a replacement. - internal void MarkHandoffTarget() - { - _knownReplacement = false; - _wasHandoffTarget = true; - } + internal void MarkHandoffTarget() => _wasHandoffTarget = true; + + /// + /// Whether this connection was made to the successor a MOVING named (see + /// ) rather than to the endpoint as resolved: recorded as the socket + /// is pointed there, so a handoff judging the connection later does not have to infer it from whether the pin + /// is still in force. Such a connection is where the server said to go, which is not the node being retired. + /// + internal bool ConnectedViaHandoffTarget => _connectedViaHandoffTarget; /// /// The socket's remote address and port, or null if the socket is not connected yet, is already gone, or - /// there is no socket. Cached once read: a connected socket's remote endpoint never changes, and the handoff's - /// bona fide test asks on every write while a hold is unresolved. + /// there is no socket. Cached once read: a connected socket's remote endpoint never changes. /// internal System.Net.IPEndPoint? TryGetRemoteEndPoint() { @@ -1104,7 +1096,7 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou // Those sync timeouts happen in ConnectionMultiplexer.ExecuteSyncImpl() via Monitor.Wait. // A command a maintenance handoff held, now written to the replacement, is covered by the // handoff for as long as that stands; every other command keeps the timeout above. - var effective = msg.HeldForHandoff ? server.GetEffectiveTimeoutMilliseconds(multiplexer.AsyncTimeoutMilliseconds, msg) : timeout; + var effective = msg.HeldForHandoff ? ServerEndPoint.GetEffectiveTimeoutMilliseconds(timeout, msg) : timeout; if (msg.HasTimedOut(now, effective, out var elapsed)) { if (msg.ResultBoxIsAsync) diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 3e48de0df..af9a40ec9 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -46,6 +46,13 @@ private MaintenanceNotificationMode MaintenanceMode ? MaintenanceNotificationMode.Disabled : Multiplexer.RawConfig.MaintenanceNotifications; + /// + /// Whether anything can change a command's timeout against this server while it waits: relaxation and the + /// handoff cover both come from notifications, so with the feature disabled a sync waiter can take its whole + /// budget in one wait rather than re-reading it. + /// + internal bool MaintenanceMayChangeTimeouts => MaintenanceMode != MaintenanceNotificationMode.Disabled; + /// /// Whether to ask this server for maintenance notifications during handshake. /// @@ -213,44 +220,24 @@ internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds) /// /// The effective timeout for a particular command: as , and - /// for a command that a maintenance handoff diverted into the backlog () no - /// less than the handoff's cover, for as long as the command lives: the cover is a property of the held command, - /// not of a window on the server, so it cannot be withdrawn between the replacement establishing and the held - /// work being written to it. Only such a command: an ordinary command on the same - /// server, including one issued by an application that has turned relaxation off and runs short timeouts, keeps - /// its configured timeout. The sweeps stop at the first command that has not timed out, so a held command at the - /// head can delay (never lose) the timeout of an ordinary one behind it, by at most the cover. - /// - internal int GetEffectiveTimeoutMilliseconds(int configuredMilliseconds, Message message) - { - var result = GetEffectiveTimeoutMilliseconds(configuredMilliseconds); - if (message.HeldForHandoff) - { - var floor = Volatile.Read(ref _holdCoverMilliseconds); - if (floor > result) result = floor; - } - return result; - } - - private int _holdCoverMilliseconds; - - /// - /// The timeout the commands a handoff hold diverts are entitled to: the hold, the rest of the window (for which a - /// MoveTo pins reconnection to the named successor) and one connect attempt, whatever the relaxation setting - a + /// for a command a maintenance handoff held () no sooner than the handoff's + /// deadline - the hold, the rest of the window and one connect attempt, whatever the relaxation setting: a /// relaxed timeout of zero is the documented way to turn relaxation off, and a sub-second configured timeout with - /// the hold's one-second floor would otherwise time out exactly the work the hold is protecting. Applies to held - /// commands only, for as long as they live; see . - /// Extends, never shortens: a handoff beginning while an earlier one's held work is still queued must not cut - /// that work's cover short. + /// the hold's one-second floor would otherwise time out exactly the work the hold is protecting. The deadline is + /// stamped on the command when it is held, so it holds for as long as the command lives and nothing that happens + /// to the handoff afterwards can withdraw it. Only such a command: an ordinary command on the same server, + /// including one issued by an application that has turned relaxation off and runs short timeouts, keeps its + /// configured timeout. /// - private void SetHoldCover(TimeSpan span) + internal static int GetEffectiveTimeoutMilliseconds(int effectiveMilliseconds, Message message) { - var ms = TickDeadline.Clamp(span); - while (true) + var deadline = message.HoldCoverDeadlineTicks; + if (deadline != 0) { - var current = Volatile.Read(ref _holdCoverMilliseconds); - if (current >= ms || Interlocked.CompareExchange(ref _holdCoverMilliseconds, ms, current) == current) return; + var floor = unchecked(deadline - message.GetWriteTime()); + if (floor > effectiveMilliseconds) effectiveMilliseconds = floor; } + return effectiveMilliseconds; } /// @@ -322,14 +309,12 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; if (tail <= TimeSpan.Zero) { - _relaxedDeadline.Clear(); - Volatile.Write(ref _relaxedEndedTicks, NudgeFromZeroTicks(Environment.TickCount)); + _relaxedDeadline.SetIn(TimeSpan.Zero); // ended now, and recorded as such for fault attribution Multiplexer.Trace($"{type}: relaxation ended", ToString()); return; } _relaxedDeadline.SetIn(tail); - Volatile.Write(ref _relaxedEndedTicks, _relaxedDeadline.Ticks); Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } @@ -351,7 +336,9 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? /// Expires, and that is not decoration. Without it, a server that named an address which turns out to be /// unreachable would pin this endpoint to it for the lifetime of the multiplexer, because every reconnect /// would keep trying the same dead target. On expiry we fall back to resolving the endpoint normally, which - /// is what we would have done anyway. + /// is what we would have done anyway. Until then it stays, whatever establishes: both bridges reconnect + /// through it, and the second one's reconnect must not be sent back to DNS - still naming the node being + /// retired - because the first one's replacement happened to establish first. /// /// internal EndPoint? HandoffTarget @@ -380,15 +367,6 @@ internal void SetHandoffTarget(EndPoint target, TimeSpan window) _handoffTarget = target; } - /// - /// Forgets any handoff target, once a connection has been established. - /// - /// - /// Called on full establishment rather than on the connect attempt: if the attempt fails we want the next - /// one to try the target again, within its window. Once a connection is up, normal resolution resumes - - /// by then DNS has usually caught up anyway. - /// - internal void ClearHandoffTarget() => _handoffTarget = null; private volatile string? _lastHandoffOutcome; /// @@ -474,7 +452,8 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint switch (decision.Action) { case HandoffAction.Recycle: - await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: false, currentEndPoint).ForAwait(); + // DNS has moved to a new address, so a reconnect reporting the old one is back on the retiring node + await DrainThenRecycleAsync(remaining, decision.Reason, replacementAddressSeen: true, currentEndPoint).ForAwait(); replaced = true; break; case HandoffAction.RecycleAtHalfWindow: @@ -482,7 +461,8 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // go on. Half of the *announced* window, less whatever the jitter already spent. var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); - await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successorPinned: false, currentEndPoint).ForAwait(); + // nothing to re-resolve, or nothing known to compare against: no second address has been seen + await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, replacementAddressSeen: false, currentEndPoint).ForAwait(); replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: @@ -491,7 +471,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining); - await DrainThenRecycleAsync(remaining, decision.Reason, successorPinned: true, currentEndPoint).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, replacementAddressSeen: true, currentEndPoint).ForAwait(); replaced = true; break; default: @@ -589,12 +569,13 @@ private TimeSpan DrainCap(TimeSpan budget) /// /// What is left of the announced window. /// Why; for the trace. - /// Whether reconnection is pinned to a named successor. If not, a connection that - /// went away by itself during the hold may have been replaced by one on the very node being retired, and that - /// replacement is retired instead; see . + /// Whether the handoff has seen a second address - the server named a + /// successor, or DNS moved to a new address - so that a reconnect reporting the retiring address is known to be + /// back on the retiring node, and is retired in its turn; see . + /// Where no second address was ever involved, the address says nothing and only time tells. /// The address and port of the node being retired, as seen from the connection that /// was told, so that a replacement which landed elsewhere is recognised as such. - private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool successorPinned, IPEndPoint? retiringEndPoint) + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool replacementAddressSeen, IPEndPoint? retiringEndPoint) { var watch = ValueStopwatch.StartNew(); // With less than the floor left of the window there is no hold worth having: the server is about to close @@ -603,11 +584,11 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su var cap = budget < DrainFloor ? TimeSpan.Zero : DrainCap(budget); // The held work, and any blocking command being waited for, must not be timed out by the hold itself, nor - // by the reconnect that follows it; the relaxed window covers it where relaxation is on, and the cover below - // covers it regardless, until every bridge's replacement is established or this span runs out. The - // reconnect can take until the end of the window plus one connect attempt: a MoveTo pins the socket target - // to the named successor for the rest of the window, and a successor that does not answer is tried until - // that expires before the endpoint is resolved the ordinary way. + // by the reconnect that follows it; the relaxed window covers it where relaxation is on, and the handoff's + // deadline below covers it regardless, stamped on each held command. The reconnect can take until the end + // of the window plus one connect attempt: a MoveTo pins the socket target to the named successor for the + // rest of the window, and a successor that does not answer is tried until that expires before the endpoint + // is resolved the ordinary way. var relax = cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout; if (relax > Multiplexer.RawConfig.MaintenanceRelaxedWindowMax) relax = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; // the user's ceiling applies here as on the notification path ExtendRelaxedWindow(relax, "MOVING handoff"); @@ -621,24 +602,20 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool su var ceiling = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; if (ceiling < drainSpan) ceiling = drainSpan; if (cover > ceiling) cover = ceiling; - SetHoldCover(cover); // snapshots: Dispose nulls the fields, and a handoff in progress must not fault on a server going away var interactiveBridge = interactive; var subscriptionBridge = subscription; - // one lifecycle for the protection: raised here with the hold, on each bridge (the FailFast shield) and on the - // server (the timeout cover); lowered only when a replacement establishes, when the handoff gives up on a - // bridge, or when this span runs out - never on a local idea of "done" along the way - // Whether a socket's remote endpoint says anything about *which node* it reached. Behind a Tunnel, or when - // the configured endpoint is itself a fixed address (a load balancer or VIP in front of the cluster), every - // connection reports the intermediary's address, and comparing it with the retiring one would make the - // genuine replacement look like a return to the retiring node forever. Then only time can tell: a connection - // established after the retire resolved is the replacement. - var identityMeaningful = Multiplexer.RawConfig.Tunnel is null - && !(EndPoint is IPEndPoint configured && retiringEndPoint is not null && configured.Equals(retiringEndPoint)); - interactiveBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned, identityMeaningful); - subscriptionBridge?.BeginQuiesce(cover, retiringEndPoint, successorPinned, identityMeaningful); + // Whether a socket's remote address says anything about *which node* it reached: only when this handoff has + // itself seen a second address, and there is no tunnel in the way. Behind a Tunnel every connection reports + // the tunnel's address; and where the only address ever involved is the one in front of the cluster (a load + // balancer or VIP) comparing it with the retiring one would make the genuine replacement look like a return + // to the retiring node forever. Then only time can tell: once the target is gone, what is current is the + // replacement. + var addressIdentifiesNode = replacementAddressSeen && Multiplexer.RawConfig.Tunnel is null; + interactiveBridge?.BeginHold(cover, retiringEndPoint, addressIdentifiesNode); + subscriptionBridge?.BeginHold(cover, retiringEndPoint, addressIdentifiesNode); // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the @@ -683,7 +660,7 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V { var force = watch.ElapsedMilliseconds >= cap.TotalMilliseconds; forced |= force; - var result = await bridge.TryRetireQuiesceTargetAsync(force, reason, lockDeadline.Token).ForAwait(); + var result = await bridge.TryRetireTargetAsync(force, reason, lockDeadline.Token).ForAwait(); if (result is { } done) { // whether we replaced the connection or it went away by itself, a replacement is now on its way, @@ -697,7 +674,7 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V // dark for the reconnect - the whole window's traffic, to protect at most one wedged writer's // message from a write fault. So, as a last resort, retire without the lock, which is what every // recycle did before handoffs existed. The protection stays up for the diverted work either way. - return new(bridge.RetireQuiesceTargetWithoutLock(reason) ? true : null, forced); + return new(bridge.RetireTargetWithoutLock(reason) ? true : null, forced); } await Task.Delay(TimeSpan.FromMilliseconds(20)).ForAwait(); } @@ -712,7 +689,7 @@ private async Task RetireBridgeAsync(PhysicalBridge? bridge, V { try { - bridge.EndQuiesce(); // idempotent; one bridge failing to release must not affect the other + bridge.EndDrain(); // idempotent; one bridge failing to release must not affect the other } catch (Exception ex) { @@ -768,23 +745,12 @@ private void ExtendRelaxedWindow(TimeSpan duration, string cause) { // a new notification never shortens an existing window: a FAILING_OVER arriving inside a longer // MIGRATING window must not cut it short - if (_relaxedDeadline.TryExtendBy(duration, out var deadline)) + if (_relaxedDeadline.TryExtendBy(duration, out _)) { - Volatile.Write(ref _relaxedEndedTicks, deadline); Multiplexer.Trace($"timeouts relaxed: {cause}", ToString()); } } - /// - /// When the most recent window was due to end, whether or not it has; zero if there has never been one. - /// - /// - /// A mirror of the deadline rather than a record of when it was cleared, which is what makes it cheap: - /// expiry is lazy (it happens on the next read), so there is no single moment at which to stamp "the - /// window closed", and the deadline already *is* that moment. - /// - private int _relaxedEndedTicks; - /// How long after a window closes a fault may still be attributed to it, at minimum. /// /// The bridge heartbeat raises timeouts on roughly a one-second cadence rather than at the deadline, so a @@ -816,7 +782,9 @@ internal MaintenanceNotificationType GetMaintenanceTypeForFault(int timeoutMilli var active = ActiveMaintenanceType; if (active != MaintenanceNotificationType.None) return active; - var ended = Volatile.Read(ref _relaxedEndedTicks); + // the deadline is kept after it passes (see TickDeadline), so it is also the record of when the most recent + // window ended: there is no single moment at which to stamp "the window closed", and the deadline is it + var ended = _relaxedDeadline.Ticks; if (ended == 0) return MaintenanceNotificationType.None; // never had a window at all var since = unchecked(Environment.TickCount - ended); diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 7b977cceb..5f0792de9 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1084,15 +1084,12 @@ internal void OnRepeatedConnectFailure(int consecutiveFailures) var last = Volatile.Read(ref _lastConnectFailureRefreshTicks); if (last != 0 && unchecked(now - last) < interval) return; - if (Interlocked.CompareExchange(ref _lastConnectFailureRefreshTicks, NudgeFromZeroTicks(now), last) != last) return; + if (Interlocked.CompareExchange(ref _lastConnectFailureRefreshTicks, Maintenance.TickDeadline.NudgeFromZero(now), last) != last) return; Multiplexer.Logger?.LogInformationRefreshingAfterConnectFailures(new(this), consecutiveFailures); Multiplexer.ReconfigureIfNeeded(EndPoint, fromBroadcast: false, $"{consecutiveFailures} consecutive connect failures"); } - /// Zero means "never", so a tick count that lands on it moves by one. - internal static int NudgeFromZeroTicks(int ticks) => ticks == 0 ? 1 : ticks; - /// /// Subscribes to the configuration-change broadcast on a RESP3 interactive connection that is shared with /// pub/sub (). Then there is no separate @@ -1126,9 +1123,6 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) // Clear the unselectable flag ASAP since we are open for business ClearUnselectable(UnselectableFlags.DidNotRespond); - // whatever a handoff pointed us at, we are connected now: resume normal resolution - ClearHandoffTarget(); - // is *this specific* connection using RESP3? (without reference to config preferences) bool isResp3 = connection?.Protocol is >= RedisProtocol.Resp3; // and if so, is pub/sub sharing it? (if not, subscriptions live on a dedicated connection) diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 57f094312..ce1603b22 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -1096,7 +1096,7 @@ public async Task MovingRecyclesTheConnectionBeforeTheServerCloses() Assert.True( await Poll.UntilAsync(() => endpoint.HandoffTarget is not null || endpoint.HandoffRecycles > 0, timeoutMilliseconds: 10_000), "the handoff should have taken the named successor"); - log.WriteLine($"handoff target: {endpoint.HandoffTarget?.ToString() ?? "(cleared after reconnect)"}; recycles={endpoint.HandoffRecycles}"); + log.WriteLine($"handoff target: {endpoint.HandoffTarget?.ToString() ?? "(expired with the window)"}; recycles={endpoint.HandoffRecycles}"); // a fresh connection re-sends the opt-in, which is how a recycle is visible from the server's side Assert.True( From 54d49e5b1d2487f0910f23150622c821f3c3f406 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 19:00:14 +0000 Subject: [PATCH 19/27] Handoff: a periodic re-judgement, doomed dominating the pin, and the drain's bounds Round 18. The phase machine's transitions were driven only by the drain loop and by establishment, so one missed instant could wedge a phase until the deadline. Every heartbeat now re-judges the current connection against the handoff in progress: a bona fide replacement resolves the hold, and a reconnect that landed back on the node being retired is adopted (once, by compare-exchange on the target) and retired in its turn, which also covers the drain loop having ended on a transient null and an address that was unreadable at establishment. The retire is idempotent under the lock. Classification puts the address before the pin, so a connection dialled through a stale pin that still named the retiring node is doomed; the via-pin fact is recorded as the address dialled and matched against the successor of the handoff doing the judging; and a MOVING that names no successor drops any earlier pin. A new hold explicitly finishes an earlier handoff still flushing, so nothing is left mid-phase. A window-closed notification arriving mid-handoff can lengthen the relaxed window but not shorten it below what the drain extended; the drain cap is bounded by the relaxation ceiling it depends on (never below the floor); the head-of-line note on BacklogEnqueue states the requeue exception honestly; and the held message's cover is a TickDeadline, extended through the one ExtendTo that TryExtendBy now uses too. --- .../Maintenance/TickDeadline.cs | 33 ++-- src/StackExchange.Redis/Message.cs | 19 +-- src/StackExchange.Redis/PhysicalBridge.cs | 146 +++++++++++++----- src/StackExchange.Redis/PhysicalConnection.cs | 17 +- .../ServerEndPoint.Maintenance.cs | 45 ++++-- 5 files changed, 180 insertions(+), 80 deletions(-) diff --git a/src/StackExchange.Redis/Maintenance/TickDeadline.cs b/src/StackExchange.Redis/Maintenance/TickDeadline.cs index 7f344b3f0..2b21f4f8a 100644 --- a/src/StackExchange.Redis/Maintenance/TickDeadline.cs +++ b/src/StackExchange.Redis/Maintenance/TickDeadline.cs @@ -11,8 +11,8 @@ namespace StackExchange.Redis.Maintenance; /// deadline that is already later. A deadline that has merely passed is kept, readable through , /// so it also records when it passed: fault attribution asks how recently the relaxed window closed, and a /// held command's cover is stamped from the handoff deadline whether or not the handoff is still running. Used for -/// the relaxed-timeout window, the successor pin and the per-bridge handoff deadline, which all had their own copies -/// of this before. +/// the relaxed-timeout window, the successor pin, the per-bridge handoff deadline and each held message's cover, which +/// all had their own copies of this before. /// internal struct TickDeadline { @@ -50,19 +50,28 @@ public static int Clamp(TimeSpan span) public bool TryExtendBy(TimeSpan span, out int deadline) { var candidate = NudgeFromZero(unchecked(Environment.TickCount + Clamp(span))); + if (ExtendTo(candidate)) + { + deadline = candidate; + return true; + } + deadline = Volatile.Read(ref _ticks); + return false; + } + + /// + /// Moves the deadline out to (a tick count, as reads) unless it is + /// already later than that; zero means "no deadline" and extends nothing. + /// + /// Whether this call extended it. + public bool ExtendTo(int ticks) + { + if (ticks == 0) return false; while (true) { var current = Volatile.Read(ref _ticks); - if (current != 0 && unchecked(candidate - current) <= 0) - { - deadline = current; - return false; - } - if (Interlocked.CompareExchange(ref _ticks, candidate, current) == current) - { - deadline = candidate; - return true; - } + if (current != 0 && unchecked(ticks - current) <= 0) return false; + if (Interlocked.CompareExchange(ref _ticks, ticks, current) == current) return true; } } diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 0177e14f1..f63d52199 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -912,28 +912,19 @@ internal void SetRequeued() /// timeout cover applies to it (and only to it: an ordinary command on the same server keeps its configured /// timeout) for as long as it lives, including after it has been written to the replacement. /// - internal bool HeldForHandoff => Volatile.Read(ref _holdCoverDeadlineTicks) != 0; + internal bool HeldForHandoff => _holdCover.Ticks != 0; /// /// The handoff's deadline, in terms, until which this message cannot time /// out: the cover is a property of the message, stamped when it is held, so nothing that happens to the handoff /// afterwards can withdraw it. Zero if the message was never held. /// - internal int HoldCoverDeadlineTicks => Volatile.Read(ref _holdCoverDeadlineTicks); + internal int HoldCoverDeadlineTicks => _holdCover.Ticks; - /// Holds the message for a handoff, covered until ; a later deadline replaces an earlier one, never the reverse. - internal void MarkHeldForHandoff(int deadlineTicks) - { - if (deadlineTicks == 0) return; // no deadline to cover it until: a hold that was over before the enqueue completed - while (true) - { - var current = Volatile.Read(ref _holdCoverDeadlineTicks); - if (current != 0 && unchecked(deadlineTicks - current) <= 0) return; - if (Interlocked.CompareExchange(ref _holdCoverDeadlineTicks, deadlineTicks, current) == current) return; - } - } + /// Holds the message for a handoff, covered until ; a later deadline replaces an earlier one, never the reverse (and zero, no deadline, covers nothing). + internal void MarkHeldForHandoff(int deadlineTicks) => _holdCover.ExtendTo(deadlineTicks); - private int _holdCoverDeadlineTicks; + private Maintenance.TickDeadline _holdCover; private int _writeTickCount; diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 95fda0916..69f01b5da 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -173,9 +173,25 @@ private enum HandoffPhase private sealed class Handoff { private int _phase; // HandoffPhase + private PhysicalConnection? _target; /// The connection the hold began on, or a reconnect adopted in its place; null if the bridge had none. - public volatile PhysicalConnection? Target; + public PhysicalConnection? Target => Volatile.Read(ref _target); + + /// + /// Adopts a reconnect that landed back on the node being retired as the target, once: of two callers that + /// see the same doomed connection (its establishment and a heartbeat, say) one adopts and retires it. + /// + public bool TryAdopt(PhysicalConnection doomed) + { + var previous = Volatile.Read(ref _target); + if (ReferenceEquals(previous, doomed) || Interlocked.CompareExchange(ref _target, doomed, previous) != previous) return false; + doomed.MarkHandoffTarget(); + return true; + } + + /// The successor this handoff pinned reconnection to, if the server named one; a connection dialled through it is the replacement. + public readonly System.Net.EndPoint? Successor; /// Whether this handoff retired a connection (rather than finding it gone). public volatile bool Retired; @@ -193,10 +209,11 @@ private sealed class Handoff /// public readonly bool AddressIdentifiesNode; - public Handoff(PhysicalConnection? target, System.Net.IPEndPoint? retiringEndPoint, bool addressIdentifiesNode) + public Handoff(PhysicalConnection? target, System.Net.IPEndPoint? retiringEndPoint, System.Net.EndPoint? successor, bool addressIdentifiesNode) { - Target = target; + _target = target; RetiringEndPoint = retiringEndPoint; + Successor = successor; AddressIdentifiesNode = addressIdentifiesNode; } @@ -285,33 +302,35 @@ private enum Replacement } /// - /// Classifies a connection against a handoff. The connection's own facts come first (rule c): the target, or a - /// reconnect adopted as one, is doomed whatever else is true of it; a connection made under the successor pin - /// went where the server said, which is not the node being retired. Then the address: where the remote address - /// identifies the node, landing back on the retiring address is doomed and anywhere else is the replacement, - /// with an unreadable address meaning unknown. Where it identifies nothing (a tunnel, a fixed front address) + /// Classifies a connection against a handoff, from the connection's own facts (rule c), doomed dominating. The + /// target, or a reconnect adopted as one, is doomed whatever else is true of it. Where the remote address + /// identifies the node and could be read, it decides: back on the retiring address is doomed - including a + /// connection dialled through a stale pin that still named that node - and anywhere else is the replacement. + /// Otherwise a connection dialled through this handoff's successor pin went where the server said, + /// which is not the node being retired. Where the address identifies nothing (a tunnel, a fixed front address) /// only time tells: while the target is still being drained a different current connection cannot be judged; - /// once the target is gone, nothing else will arrive, so what is current is the replacement. + /// once the target is gone, nothing else will arrive, so what is current is the replacement. What is left is + /// an address that could not be read yet, which is unknown and defers to a later pass (rule b) - unless there + /// is nothing to compare it against anyway, in which case the connection is taken at its word rather than + /// held to the deadline with nothing ever able to judge it. /// private static Replacement Classify(Handoff handoff, PhysicalConnection connection) { if (connection.WasHandoffTarget) return Replacement.Doomed; - if (connection.ConnectedViaHandoffTarget) return Replacement.Yes; - if (!handoff.AddressIdentifiesNode) + var retiring = handoff.RetiringEndPoint; + if (handoff.AddressIdentifiesNode && retiring is not null && connection.TryGetRemoteEndPoint() is { } landedOn) { - return handoff.Phase == HandoffPhase.Holding ? Replacement.Unknown : Replacement.Yes; + return landedOn.Equals(retiring) ? Replacement.Doomed : Replacement.Yes; } - var retiring = handoff.RetiringEndPoint; - if (retiring is null) + if (connection.HandoffTargetDialed is { } dialed && handoff.Successor is { } successor && dialed.Equals(successor)) { - // nothing to compare against: the connection that was told had no readable address. Deferring would - // hold traffic until the deadline with nothing ever able to classify, so the replacement is taken at - // its word return Replacement.Yes; } - var landedOn = connection.TryGetRemoteEndPoint(); - if (landedOn is null) return Replacement.Unknown; - return landedOn.Equals(retiring) ? Replacement.Doomed : Replacement.Yes; + if (!handoff.AddressIdentifiesNode) + { + return handoff.Phase == HandoffPhase.Holding ? Replacement.Unknown : Replacement.Yes; + } + return retiring is null ? Replacement.Yes : Replacement.Unknown; } /// Whether the backlog still holds work a handoff diverted; the protection stands until it has been written (rule a). @@ -351,36 +370,79 @@ private bool IsHandoffOwnFailure(PhysicalConnection connection) /// cover on the held commands - if no bona fide replacement establishes and flushes first. /// The address and port of the node being retired, as seen from the connection /// that was told, if known; see . + /// The successor the server named and reconnection is pinned to, if any. /// Whether a socket's remote address identifies the node it reached; see /// . - internal void BeginHold(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, bool addressIdentifiesNode) + internal void BeginHold(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, System.Net.EndPoint? successor, bool addressIdentifiesNode) { var target = physical; target?.MarkHandoffTarget(); + // An earlier handoff still flushing is finished here, explicitly: its replacement is the connection this + // one retires, so its flush cannot complete, and its held work - which belongs to the bridge's backlog, + // not to a handoff - keeps its stamped cover and is flushed with this handoff's. Nothing is left mid-phase. + _handoff?.Finish(); // the deadline before the phase: a reader that sees the new handoff must also see a live deadline, or it // would end it on the spot. Extended, never reset: an earlier handoff's held work may still be queued. _handoffDeadline.TryExtendBy(protection, out _); - _handoff = new Handoff(target, retiringEndPoint, addressIdentifiesNode); + _handoff = new Handoff(target, retiringEndPoint, successor, addressIdentifiesNode); } /// /// The drain loop has ended, whatever it achieved. A target it could not retire (the lock never came) is left - /// to the server's close, and the hold stays until a replacement is current; a replacement that established + /// to the server's close, and the hold stays until a replacement is current; a connection that established /// while the drain was still judging the target is judged again now that the target is gone. /// internal void EndDrain() { if (!TryGetHandoff(out var handoff)) return; handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); - ResolveIfReplacementCurrent(handoff); + ReevaluateHandoff(); } - /// Resolves the hold if the current connection is an established bona fide replacement. - private void ResolveIfReplacementCurrent(Handoff handoff) + /// + /// Judges the current connection against the handoff in progress, if it is established. Run at establishment, + /// when the drain loop ends, and from every heartbeat: the drain loop and the establishment callback each see + /// one instant, and a phase must not be wedged by what that instant happened to miss - a poll landing in the + /// gap between a disconnect and the reconnect, or an address that could not be read at the moment of + /// establishment (rule b: unknown defers to a later pass, and the heartbeat is that pass). + /// + private void ReevaluateHandoff() { - if (physical is { EverFullyEstablished: true } current && Classify(handoff, current) == Replacement.Yes) + if (!TryGetHandoff(out var handoff) || handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return; + if (physical is { EverFullyEstablished: true } current) Judge(handoff, current); + } + + /// + /// An established connection, judged: a bona fide replacement resolves the hold; a reconnect that landed back + /// on the node being retired, once the target is gone, is adopted and retired in its turn, as the drain loop + /// would have retired it. The retire takes the write lock, so it runs apart from the caller (an establishment + /// callback, or the heartbeat); while the target is still being drained, the drain loop owns the judgement. + /// + private void Judge(Handoff handoff, PhysicalConnection connection) + { + switch (Classify(handoff, connection)) { - OnReplacementEstablished(handoff); + case Replacement.Yes: + OnReplacementEstablished(handoff); + break; + case Replacement.Doomed when handoff.Phase == HandoffPhase.AwaitingReplacement && !ReferenceEquals(connection, handoff.Target): + // force: nothing a caller is waiting on can be in flight on it - the hold has diverted caller work + // since before it was connected - and a pass that waited for one would have no second pass + if (handoff.TryAdopt(connection)) _ = RetireAdoptedAsync(); + break; + } + } + + private async Task RetireAdoptedAsync() + { + try + { + using var lockDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + await TryRetireTargetAsync(force: true, "reconnected to the node being retired", lockDeadline.Token).ForAwait(); + } + catch (Exception ex) + { + OnInternalError(ex); } } @@ -520,13 +582,15 @@ private static bool ShouldHoldBatch(List messages) // never branded a target; the next pass looks again, and the drain's deadline bounds this return null; default: - // back on the node being retired: what is current now is what the handoff replaces - current.MarkHandoffTarget(); - handoff.Target = current; + // back on the node being retired: what is current now is what the handoff replaces - + // unless its establishment or a heartbeat adopted it first, in which case that retire is + // under way and the next pass sees its result + if (!handoff.TryAdopt(current)) return null; toRetire = current; break; } } + if (toRetire.RetiredForHandoff) return handoff.Retired; // already retired by another path; its teardown follows that path's lock release if (!force && toRetire.HasCallerMessagesAwaitingResponse()) return null; // Under the lock only the marking: from here no writer that takes the lock writes to the connection (see @@ -1049,8 +1113,9 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) LastException = null; // only a bona fide replacement establishing ends the hold for this bridge - not the very connection // the hold is draining completing a handshake it was in the middle of when the hold began, and not a - // reconnect that landed back on the node being retired (rule c: the connection's own facts decide) - if (TryGetHandoff(out var handoff) && Classify(handoff, connection) == Replacement.Yes) OnReplacementEstablished(handoff); + // reconnect that landed back on the node being retired, which is retired in its turn (rule c: the + // connection's own facts decide) + if (TryGetHandoff(out var handoff)) Judge(handoff, connection); Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); @@ -1091,6 +1156,9 @@ internal void OnHeartbeat(bool ifConnectedOnly) StartBacklogProcessor(); } + // a handoff in progress is judged again on every beat, so that no phase is wedged by one missed instant + ReevaluateHandoff(); + runThisTime = !isDisposed && Interlocked.CompareExchange(ref beating, 1, 0) == 0; if (!runThisTime) return; @@ -1490,10 +1558,14 @@ internal void AcceptRerouted(Message message) /// The one place a message enters the backlog, and so the one place the hold's cover is applied: whatever /// enters during a handoff is held - diverted off the live connection, queued while the bridge reconnects to /// the replacement, or parked behind held work being flushed - and gets the same cover, to the same deadline. - /// The shared deadline is also what keeps the timeout sweeps' head-of-line rule sound: a message enqueued - /// later never times out before one enqueued earlier, because an earlier held message expires at the same - /// deadline or sooner, and an ordinary message can only be enqueued after the handoff is done, which is when - /// nothing held is left or that deadline has passed. + /// The shared deadline also keeps the timeout sweeps' head-of-line rule sound for everything enqueued fresh: a + /// message enqueued later does not time out before one enqueued earlier, because an earlier held message + /// expires at the same deadline or sooner, and an ordinary message can only be enqueued after the handoff is + /// done, which is when nothing held is left or that deadline has passed. The one exception is a message + /// requeued by a writer that reached the lock after the retire: it keeps its older write time (the budget it + /// spent counts) behind heads with newer ones, so where its own timeout rather than the shared deadline is the + /// later of the two it can be reported late - by at most the heads' remaining budget - never early, and never + /// lost. /// /// The message to queue. /// The message was diverted by a handoff's hold (or requeued after the retire): held diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index ef9d76cd1..1068fab6f 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -316,7 +316,7 @@ internal async Task BeginConnectAsync(ILogger? log) { Trace($"handoff: connecting to {Format.ToString(handoffTarget)} in place of {Format.ToString(endpoint)}"); connectTo = handoffTarget; - _connectedViaHandoffTarget = true; + _handoffTargetDialed = handoffTarget; } if (tunnel is not null) { @@ -989,7 +989,8 @@ internal void GetStormLog(StringBuilder sb) // volatile: each is written on one thread and read on others as part of the handoff's identity test, in the // very races that test exists for; a plain field could be read stale exactly then - private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished, _connectedViaHandoffTarget; + private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished; + private volatile System.Net.EndPoint? _handoffTargetDialed; /// /// Whether this connection was ever the one a maintenance handoff held work back from. A durable fact about the @@ -1002,12 +1003,14 @@ internal void GetStormLog(StringBuilder sb) internal void MarkHandoffTarget() => _wasHandoffTarget = true; /// - /// Whether this connection was made to the successor a MOVING named (see - /// ) rather than to the endpoint as resolved: recorded as the socket - /// is pointed there, so a handoff judging the connection later does not have to infer it from whether the pin - /// is still in force. Such a connection is where the server said to go, which is not the node being retired. + /// The successor a MOVING named (see ) that this connection was + /// dialled to, in place of the endpoint as resolved; null if it was dialled to the endpoint. Recorded as the + /// socket is pointed there, so a handoff judging the connection later does not have to infer it from whether + /// the pin is still in force - and recorded as the address rather than a flag, so that the judgement can be + /// scoped to the handoff whose successor it is: a connection dialled through an earlier handoff's pin is not + /// thereby the replacement for a later one. /// - internal bool ConnectedViaHandoffTarget => _connectedViaHandoffTarget; + internal System.Net.EndPoint? HandoffTargetDialed => _handoffTargetDialed; /// /// The socket's remote address and port, or null if the socket is not connected yet, is already gone, or diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index af9a40ec9..9f24a03b0 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -307,14 +307,21 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? Volatile.Write(ref _relaxedType, (int)type); var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; + + // While a handoff is in flight, the drain depends on the relaxation it extended: a completion for some other + // operation arriving mid-drain (a MIGRATED for a shard move) must not pull the deadline back under the drain's + // cap and have the sweep fail the very commands the drain is waiting for. So during a handoff a close can + // lengthen the window but not shorten it; the replace semantics above apply the rest of the time. + var handoffInFlight = Volatile.Read(ref _handoffInFlight) != 0; if (tail <= TimeSpan.Zero) { - _relaxedDeadline.SetIn(TimeSpan.Zero); // ended now, and recorded as such for fault attribution + if (!handoffInFlight) _relaxedDeadline.SetIn(TimeSpan.Zero); // ended now, and recorded as such for fault attribution Multiplexer.Trace($"{type}: relaxation ended", ToString()); return; } - _relaxedDeadline.SetIn(tail); + if (handoffInFlight) _relaxedDeadline.TryExtendBy(tail, out _); + else _relaxedDeadline.SetIn(tail); Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } @@ -367,6 +374,12 @@ internal void SetHandoffTarget(EndPoint target, TimeSpan window) _handoffTarget = target; } + /// + /// Drops the pin when a later MOVING names no successor: the pinned node is then, as likely as not, the + /// very one now being retired, and a reconnect dialled through the stale pin would land on it. + /// + private void ClearHandoffTarget() => _handoffTarget = null; + private volatile string? _lastHandoffOutcome; /// @@ -452,8 +465,10 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint switch (decision.Action) { case HandoffAction.Recycle: - // DNS has moved to a new address, so a reconnect reporting the old one is back on the retiring node - await DrainThenRecycleAsync(remaining, decision.Reason, replacementAddressSeen: true, currentEndPoint).ForAwait(); + // DNS has moved to a new address, so a reconnect reporting the old one is back on the retiring + // node; a pin left by an earlier handoff is stale now and must not dial it + ClearHandoffTarget(); + await DrainThenRecycleAsync(remaining, decision.Reason, successor: null, replacementAddressSeen: true, currentEndPoint).ForAwait(); replaced = true; break; case HandoffAction.RecycleAtHalfWindow: @@ -462,7 +477,8 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); // nothing to re-resolve, or nothing known to compare against: no second address has been seen - await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, replacementAddressSeen: false, currentEndPoint).ForAwait(); + ClearHandoffTarget(); + await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successor: null, replacementAddressSeen: false, currentEndPoint).ForAwait(); replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: @@ -471,7 +487,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining); - await DrainThenRecycleAsync(remaining, decision.Reason, replacementAddressSeen: true, currentEndPoint).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason, target, replacementAddressSeen: true, currentEndPoint).ForAwait(); replaced = true; break; default: @@ -502,7 +518,8 @@ private static TimeSpan Remaining(TimeSpan window, ValueStopwatch watch) /// /// How long a handoff holds new work and waits for in-flight commands before replacing the connections /// regardless: half the relaxed timeout (five seconds by default), never more than half of what is left of the - /// window, never less than , and never more than the window. + /// window, never less than , never more than the relaxation ceiling (or the floor, if + /// that is higher), and never more than the window. /// /// /// Two things pull in opposite directions. A blocking command in flight - BLPOP, XREAD BLOCK, WAIT - is failed @@ -523,6 +540,12 @@ private TimeSpan DrainCap(TimeSpan budget) var halfWindow = TimeSpan.FromTicks(budget.Ticks / 2); if (cap > halfWindow) cap = halfWindow; if (cap < DrainFloor) cap = DrainFloor; // a relaxed timeout of zero disables relaxation; it must not disable the drain + // the relaxation the drain depends on is itself bounded by the user's ceiling: holding for longer than the + // sweep will relax would pay the hold's latency and still fail the command. The floor still applies, as for a + // relaxed timeout of zero: a sub-second ceiling says "do not relax", not "do not drain". + var ceiling = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; + if (ceiling < DrainFloor) ceiling = DrainFloor; + if (cap > ceiling) cap = ceiling; return cap > budget ? budget : cap; } @@ -569,13 +592,15 @@ private TimeSpan DrainCap(TimeSpan budget) /// /// What is left of the announced window. /// Why; for the trace. + /// The successor the server named and reconnection is pinned to, if any: a connection + /// dialled through it is the replacement. /// Whether the handoff has seen a second address - the server named a /// successor, or DNS moved to a new address - so that a reconnect reporting the retiring address is known to be /// back on the retiring node, and is retired in its turn; see . /// Where no second address was ever involved, the address says nothing and only time tells. /// The address and port of the node being retired, as seen from the connection that /// was told, so that a replacement which landed elsewhere is recognised as such. - private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool replacementAddressSeen, IPEndPoint? retiringEndPoint) + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, EndPoint? successor, bool replacementAddressSeen, IPEndPoint? retiringEndPoint) { var watch = ValueStopwatch.StartNew(); // With less than the floor left of the window there is no hold worth having: the server is about to close @@ -614,8 +639,8 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, bool re // to the retiring node forever. Then only time can tell: once the target is gone, what is current is the // replacement. var addressIdentifiesNode = replacementAddressSeen && Multiplexer.RawConfig.Tunnel is null; - interactiveBridge?.BeginHold(cover, retiringEndPoint, addressIdentifiesNode); - subscriptionBridge?.BeginHold(cover, retiringEndPoint, addressIdentifiesNode); + interactiveBridge?.BeginHold(cover, retiringEndPoint, successor, addressIdentifiesNode); + subscriptionBridge?.BeginHold(cover, retiringEndPoint, successor, addressIdentifiesNode); // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the From 60da7a6b6bb11736d6904ea5aff6a2883c538e93 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 19:28:52 +0000 Subject: [PATCH 20/27] Handoff: per-handoff deadlines, a second chance for the retire, and the last two wrap hazards Round 19. The handoff deadline moves onto the handoff object: each handoff stands for its own hold plus the rest of its window and one connect attempt, held work is stamped with that, and the deadline is read on every question put to a live handoff, so a passed one is noticed fresh and no stale residue is ever consulted by a later handoff. BeginHold publishes the new handoff before finishing the previous one, so there is no instant with no hold in force. The heartbeat's judgement now also retires the target itself when it is still current after the drain loop is over (the lock never came, or the retire faulted), not only an adopted reconnect; the retire outside the loop is one at a time per handoff and idempotent under the lock, so a stuck attempt is not piled on and a failed one is retried on the next beat. TickDeadline treats a deadline as in force only within one span of now, in both RemainingMilliseconds and ExtendTo, so a wrapped tick count can neither revive a long-passed deadline nor let it win against a fresh extension; the successor pin is read on every server heartbeat so its expiry is noticed while fresh. Sync waiters slice only where the server has ever accepted the opt-in, has a window open, or has a handoff in flight. The steady-load test helper's try body is indented. --- .../Maintenance/TickDeadline.cs | 21 +++-- src/StackExchange.Redis/PhysicalBridge.cs | 83 ++++++++++++------- .../ServerEndPoint.Maintenance.cs | 24 ++++-- src/StackExchange.Redis/ServerEndPoint.cs | 1 + .../MaintenanceNotificationTests.cs | 72 ++++++++-------- 5 files changed, 122 insertions(+), 79 deletions(-) diff --git a/src/StackExchange.Redis/Maintenance/TickDeadline.cs b/src/StackExchange.Redis/Maintenance/TickDeadline.cs index 2b21f4f8a..b230cbf25 100644 --- a/src/StackExchange.Redis/Maintenance/TickDeadline.cs +++ b/src/StackExchange.Redis/Maintenance/TickDeadline.cs @@ -6,9 +6,9 @@ namespace StackExchange.Redis.Maintenance; /// /// A deadline expressed in terms, where zero means "unset". One place for the /// three invariants every such field needs: a genuine zero tick count is nudged to one so it cannot read as unset; -/// a long-passed deadline is cleared (by compare-exchange, so a concurrent extension is not undone), so that a tick -/// count wrapping ~24.8 days later cannot bring it back to life; and extension is monotonic, never shortening a -/// deadline that is already later. A deadline that has merely passed is kept, readable through , +/// a deadline is in force only within one span () of now, so the residue of one that +/// passed long ago cannot come back to life when the tick count wraps ~24.8 days later, nor win a comparison against +/// a fresh extension; and extension is monotonic, never shortening a deadline that is already later. A deadline that has merely passed is kept, readable through , /// so it also records when it passed: fault attribution asks how recently the relaxed window closed, and a /// held command's cover is stamped from the handoff deadline whether or not the handoff is still running. Used for /// the relaxed-timeout window, the successor pin, the per-bridge handoff deadline and each held message's cover, which @@ -67,10 +67,15 @@ public bool TryExtendBy(TimeSpan span, out int deadline) public bool ExtendTo(int ticks) { if (ticks == 0) return false; + var now = Environment.TickCount; while (true) { var current = Volatile.Read(ref _ticks); - if (current != 0 && unchecked(ticks - current) <= 0) return false; + // a deadline in force is within one span of now, either way; anything else is the residue of a deadline + // that passed so long ago that the tick count has wrapped past it, and must not win a comparison + var ahead = unchecked(current - now); + var inForce = current != 0 && ahead > -MaxMilliseconds && ahead <= MaxMilliseconds; + if (inForce && unchecked(ticks - current) <= 0) return false; if (Interlocked.CompareExchange(ref _ticks, ticks, current) == current) return true; } } @@ -84,10 +89,12 @@ public int RemainingMilliseconds() var deadline = Volatile.Read(ref _ticks); if (deadline == 0) return 0; var remaining = unchecked(deadline - Environment.TickCount); - if (remaining > 0) return remaining; + if (remaining > 0 && remaining <= MaxMilliseconds) return remaining; - // long passed: clear it, but only if nobody has moved it on in the meantime - if (remaining < -MaxMilliseconds) Interlocked.CompareExchange(ref _ticks, 0, deadline); + // Long passed, or further ahead than any span this type sets - which is what a deadline that passed more + // than half a wrap ago looks like once the tick count has come round: clear it, but only if nobody has moved + // it on in the meantime. + if (remaining < -MaxMilliseconds || remaining > MaxMilliseconds) Interlocked.CompareExchange(ref _ticks, 0, deadline); return 0; } diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 69f01b5da..ccc776855 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -173,8 +173,18 @@ private enum HandoffPhase private sealed class Handoff { private int _phase; // HandoffPhase + private int _retiring; // a retire of the target, outside the drain loop, is under way private PhysicalConnection? _target; + /// + /// Until when this handoff stands at the latest: the hold, the rest of the window and one connect attempt, + /// from when the hold began. This handoff's own, not shared with an earlier one: held work is stamped with + /// it, so the time a command can be in limbo is bounded by the handoff that held it, never by a longer one + /// that happened to come before. It is read on every question put to a live handoff, so a passed deadline + /// is noticed while it is still fresh. + /// + public Maintenance.TickDeadline Deadline; + /// The connection the hold began on, or a reconnect adopted in its place; null if the bridge had none. public PhysicalConnection? Target => Volatile.Read(ref _target); @@ -209,14 +219,20 @@ public bool TryAdopt(PhysicalConnection doomed) /// public readonly bool AddressIdentifiesNode; - public Handoff(PhysicalConnection? target, System.Net.IPEndPoint? retiringEndPoint, System.Net.EndPoint? successor, bool addressIdentifiesNode) + public Handoff(PhysicalConnection? target, System.Net.IPEndPoint? retiringEndPoint, System.Net.EndPoint? successor, bool addressIdentifiesNode, TimeSpan protection) { _target = target; RetiringEndPoint = retiringEndPoint; Successor = successor; AddressIdentifiesNode = addressIdentifiesNode; + Deadline.SetIn(protection); } + /// Claims the one-at-a-time retire of the target outside the drain loop; see . + public bool TryBeginRetire() => Interlocked.CompareExchange(ref _retiring, 1, 0) == 0; + + public void EndRetire() => Volatile.Write(ref _retiring, 0); + public HandoffPhase Phase => (HandoffPhase)Volatile.Read(ref _phase); /// Advances the phase, if it is still where the caller saw it. @@ -229,21 +245,12 @@ public bool TryAdvance(HandoffPhase from, HandoffPhase to) private volatile Handoff? _handoff; - /// - /// Until when a handoff's protection can stand at the latest: the hold, the rest of the window and one connect - /// attempt. Shared by successive handoffs and only ever extended, since an earlier handoff's held work may - /// still be queued when a later one begins. Its ticks also stamp each held message's timeout cover, so it is - /// left to lapse rather than cleared when a handoff finishes early: the cover is the message's, for its whole - /// life, and must not be withdrawn between the replacement establishing and the message being answered. - /// - private Maintenance.TickDeadline _handoffDeadline; - /// The handoff in progress, if there is one and its deadline has not passed (a passed deadline ends it). private bool TryGetHandoff([NotNullWhen(true)] out Handoff? handoff) { handoff = _handoff; if (handoff is null || handoff.Phase == HandoffPhase.Done) return false; - if (_handoffDeadline.IsActive) return true; + if (handoff.Deadline.IsActive) return true; handoff.Finish(); return false; } @@ -377,14 +384,15 @@ internal void BeginHold(TimeSpan protection, System.Net.IPEndPoint? retiringEndP { var target = physical; target?.MarkHandoffTarget(); - // An earlier handoff still flushing is finished here, explicitly: its replacement is the connection this - // one retires, so its flush cannot complete, and its held work - which belongs to the bridge's backlog, - // not to a handoff - keeps its stamped cover and is flushed with this handoff's. Nothing is left mid-phase. - _handoff?.Finish(); - // the deadline before the phase: a reader that sees the new handoff must also see a live deadline, or it - // would end it on the spot. Extended, never reset: an earlier handoff's held work may still be queued. - _handoffDeadline.TryExtendBy(protection, out _); - _handoff = new Handoff(target, retiringEndPoint, successor, addressIdentifiesNode); + // Publish the new handoff, then finish the previous one - in that order, so that there is no instant at + // which neither hold is in force: a wake racing this would otherwise flush the previous hold's work to the + // very connection this hold is about to drain. The previous handoff (still flushing, say) is finished + // explicitly rather than left mid-phase: its replacement is the connection this one retires, so its flush + // cannot complete, and its held work - which belongs to the bridge's backlog, not to a handoff - keeps its + // stamped cover and is flushed with this handoff's. + var previous = _handoff; + _handoff = new Handoff(target, retiringEndPoint, successor, addressIdentifiesNode, protection); + previous?.Finish(); } /// @@ -409,14 +417,17 @@ internal void EndDrain() private void ReevaluateHandoff() { if (!TryGetHandoff(out var handoff) || handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return; - if (physical is { EverFullyEstablished: true } current) Judge(handoff, current); + if (physical is { EverFullyEstablished: true, RetiredForHandoff: false } current) Judge(handoff, current); } /// - /// An established connection, judged: a bona fide replacement resolves the hold; a reconnect that landed back - /// on the node being retired, once the target is gone, is adopted and retired in its turn, as the drain loop - /// would have retired it. The retire takes the write lock, so it runs apart from the caller (an establishment - /// callback, or the heartbeat); while the target is still being drained, the drain loop owns the judgement. + /// An established connection, judged: a bona fide replacement resolves the hold; a doomed one, once the drain + /// loop is over, is retired - a reconnect that landed back on the node being retired (adopted first, as the + /// drain loop would have adopted it), or the target itself, still current because the loop's retire never got + /// the lock or faulted. The retire takes the write lock, so it runs apart from the caller (an establishment + /// callback, or the heartbeat), one at a time and idempotent under the lock: a heartbeat that finds the + /// previous attempt still waiting does not pile on, and one that finds it failed tries again. While the target + /// is still being drained, the drain loop owns the judgement. /// private void Judge(Handoff handoff, PhysicalConnection connection) { @@ -425,25 +436,35 @@ private void Judge(Handoff handoff, PhysicalConnection connection) case Replacement.Yes: OnReplacementEstablished(handoff); break; - case Replacement.Doomed when handoff.Phase == HandoffPhase.AwaitingReplacement && !ReferenceEquals(connection, handoff.Target): - // force: nothing a caller is waiting on can be in flight on it - the hold has diverted caller work - // since before it was connected - and a pass that waited for one would have no second pass - if (handoff.TryAdopt(connection)) _ = RetireAdoptedAsync(); + case Replacement.Doomed when handoff.Phase == HandoffPhase.AwaitingReplacement: + if ((ReferenceEquals(connection, handoff.Target) || handoff.TryAdopt(connection)) && handoff.TryBeginRetire()) + { + _ = RetireTargetAsync(handoff); + } break; } } - private async Task RetireAdoptedAsync() + /// + /// The retire outside the drain loop. Forced: the loop's own forced pass has come and gone by the time this + /// runs, and nothing a caller is waiting on can be in flight on a reconnect that the hold has diverted caller + /// work away from since before it was connected. + /// + private async Task RetireTargetAsync(Handoff handoff) { try { using var lockDeadline = new CancellationTokenSource(TimeSpan.FromSeconds(1)); - await TryRetireTargetAsync(force: true, "reconnected to the node being retired", lockDeadline.Token).ForAwait(); + await TryRetireTargetAsync(force: true, "still on the node being retired after the drain", lockDeadline.Token).ForAwait(); } catch (Exception ex) { OnInternalError(ex); } + finally + { + handoff.EndRetire(); + } } /// @@ -1574,7 +1595,7 @@ internal void AcceptRerouted(Message message) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void BacklogEnqueue(Message message, bool diverted = false) { - if (diverted || Phase != HandoffPhase.Done) message.MarkHeldForHandoff(_handoffDeadline.Ticks); + if (_handoff is { } handoff && (diverted || TryGetHandoff(out _))) message.MarkHeldForHandoff(handoff.Deadline.Ticks); bool wasEmpty = !BacklogHasItems; // important that this *precedes* enqueue, to play well with HasPendingCallerFacingItems Interlocked.Increment(ref _backlogCurrentEnqueued); diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 9f24a03b0..a35dec4e5 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -47,11 +47,17 @@ private MaintenanceNotificationMode MaintenanceMode : Multiplexer.RawConfig.MaintenanceNotifications; /// - /// Whether anything can change a command's timeout against this server while it waits: relaxation and the - /// handoff cover both come from notifications, so with the feature disabled a sync waiter can take its whole - /// budget in one wait rather than re-reading it. + /// Whether anything can change a command's timeout against this server while it waits, so that a sync waiter + /// knows whether to re-read its budget or take it in one wait. Relaxation and the handoff cover both come from + /// notifications, so: a server that has ever accepted the opt-in (the negotiation record, kept across handshakes + /// because the current connection's is cleared at the start of each one and a notification can arrive the moment + /// a handshake completes), a window already open (which can still close, shrinking the budget back), or a handoff + /// in flight. A server that refused, or was never asked, cannot change anything. /// - internal bool MaintenanceMayChangeTimeouts => MaintenanceMode != MaintenanceNotificationMode.Disabled; + internal bool MaintenanceMayChangeTimeouts + => _maintenanceNotificationsEverAccepted || GetRelaxedRemaining() > 0 || Volatile.Read(ref _handoffInFlight) != 0; + + private volatile bool _maintenanceNotificationsEverAccepted; /// /// Whether to ask this server for maintenance notifications during handshake. @@ -109,7 +115,7 @@ private RedisValue MaintenanceMovingEndpointTypeLiteral(PhysicalConnection conne internal void OnMaintenanceNotificationsAccepted(PhysicalConnection connection) { - _maintenanceNotificationsActive = true; + _maintenanceNotificationsActive = _maintenanceNotificationsEverAccepted = true; Multiplexer.Logger?.LogInformationMaintenanceNotificationsAccepted(new(this)); } @@ -380,6 +386,14 @@ internal void SetHandoffTarget(EndPoint target, TimeSpan window) /// private void ClearHandoffTarget() => _handoffTarget = null; + /// + /// Reads the pin on every heartbeat so that its expiry is noticed while it is fresh. Expiry is lazy, on read, + /// and the connect path is the only other reader: a server that stays connected for weeks after a handoff would + /// otherwise carry a months-old expiry into its next reconnect, where a wrapped tick count could read it as in + /// force again. + /// + internal void ExpireHandoffTargetIfDue() => _ = HandoffTarget; + private volatile string? _lastHandoffOutcome; /// diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 5f0792de9..4f092af03 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -1235,6 +1235,7 @@ internal void OnHeartbeat() { try { + ExpireHandoffTargetIfDue(); interactive?.OnHeartbeat(false); subscription?.OnHeartbeat(false); } diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index ce1603b22..460c7a6dd 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -908,43 +908,43 @@ private async Task SteadyLoadHandoffAsync(Action? configur })).ToArray(); try { - // 20 completions can take a while when the workload is deliberately slow and served one at a time - Assert.True(await Poll.UntilAsync(() => Volatile.Read(ref completed) > 20, timeoutMilliseconds: 30_000), "the load should be flowing before the handoff"); + // 20 completions can take a while when the workload is deliberately slow and served one at a time + Assert.True(await Poll.UntilAsync(() => Volatile.Read(ref completed) > 20, timeoutMilliseconds: 30_000), "the load should be flowing before the handoff"); - var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); - var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); - var watch = System.Diagnostics.Stopwatch.StartNew(); - server.SendMoving(null, timeSeconds: 10, newEndpoint: successor, sequenceId: 0); - var moving = await events.NextAsync(); - Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); - - Assert.True( - await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8_000), - "the handoff should have replaced the connection"); - var recycledAfter = watch.Elapsed; - log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms of a 10s window"); - - // jitter is at most a tenth of the window and the drain is one round trip; recycling at the deadline is the bug - Assert.True(recycledAfter < TimeSpan.FromSeconds(5), $"recycled after {recycledAfter}: that is waiting for the window, not draining"); - - // the replacement carries the held work and whatever follows. Note the in-process transport never dials the - // successor: its tunnel derives the socket target from the configured endpoint, so the reconnect goes - // straight back to this server; the timing here is the hold and the handshake, not a refused address. - var resumeWatch = System.Diagnostics.Stopwatch.StartNew(); - var before = Volatile.Read(ref completed); - Assert.True( - await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutMilliseconds: 20_000), - "traffic should resume on the replacement connection"); - log.WriteLine($"traffic resumed {resumeWatch.ElapsedMilliseconds}ms after the recycle was observed"); - - stop.Cancel(); - await Task.WhenAll(workerTasks); - lock (failures) - { - log.WriteLine($"completed: {Volatile.Read(ref completed)}; failed: {failures.Count}"); - Assert.Empty(failures.Select(f => f.Message)); - } - Assert.DoesNotContain(logs.Lines, l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + var successor = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + var watch = System.Diagnostics.Stopwatch.StartNew(); + server.SendMoving(null, timeSeconds: 10, newEndpoint: successor, sequenceId: 0); + var moving = await events.NextAsync(); + Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); + + Assert.True( + await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8_000), + "the handoff should have replaced the connection"); + var recycledAfter = watch.Elapsed; + log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms of a 10s window"); + + // jitter is at most a tenth of the window and the drain is one round trip; recycling at the deadline is the bug + Assert.True(recycledAfter < TimeSpan.FromSeconds(5), $"recycled after {recycledAfter}: that is waiting for the window, not draining"); + + // the replacement carries the held work and whatever follows. Note the in-process transport never dials the + // successor: its tunnel derives the socket target from the configured endpoint, so the reconnect goes + // straight back to this server; the timing here is the hold and the handshake, not a refused address. + var resumeWatch = System.Diagnostics.Stopwatch.StartNew(); + var before = Volatile.Read(ref completed); + Assert.True( + await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutMilliseconds: 20_000), + "traffic should resume on the replacement connection"); + log.WriteLine($"traffic resumed {resumeWatch.ElapsedMilliseconds}ms after the recycle was observed"); + + stop.Cancel(); + await Task.WhenAll(workerTasks); + lock (failures) + { + log.WriteLine($"completed: {Volatile.Read(ref completed)}; failed: {failures.Count}"); + Assert.Empty(failures.Select(f => f.Message)); + } + Assert.DoesNotContain(logs.Lines, l => l.Contains("did not establish a replacement", StringComparison.Ordinal)); } finally { From 99a9ab8a9a7fbea58f12941503ed6ed1a6db17f1 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 20:19:36 +0000 Subject: [PATCH 21/27] Handoff: edges - same-address successors, sweeps past covered heads, retire and stamp races, slicing only while a window is open Round 20. A successor at the very address being retired (a proxy restarted in place) no longer makes the address identify the node, and a connection dialled through this handoff's own pin is the replacement before the address is consulted - the stale-pin case the address-first order served is covered by matching the pin against the judging handoff's successor and by a no-successor MOVING dropping the pin. The written-queue sweep looks past a held head rather than stopping at it, as the backlog's ordering argument never covered the replacement's queue. BacklogEnqueue stamps from the same handoff that gated the message, and not for internal traffic. Marking a connection retired is a compare-exchange, so a lockless last-resort retire and a locked one tear it down once between them. The processor's in-hand message counts as queued until written, so a flush cannot be declared finished around it. The one-at-a-time guard on a MOVING covers the decision and the drain, not the warning wait, so a second MOVING inside the first window is acted on. Sync waiters slice only while a window is open or a handoff is in flight; an extension is honoured by the re-read after the wait regardless. TickDeadline.cs gets the repository's UTF-8 BOM. --- .../Maintenance/TickDeadline.cs | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 64 +++++++++++++------ src/StackExchange.Redis/PhysicalConnection.cs | 15 ++++- .../ServerEndPoint.Maintenance.cs | 36 ++++++----- 4 files changed, 77 insertions(+), 40 deletions(-) diff --git a/src/StackExchange.Redis/Maintenance/TickDeadline.cs b/src/StackExchange.Redis/Maintenance/TickDeadline.cs index b230cbf25..42e0e6f7e 100644 --- a/src/StackExchange.Redis/Maintenance/TickDeadline.cs +++ b/src/StackExchange.Redis/Maintenance/TickDeadline.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; namespace StackExchange.Redis.Maintenance; diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index ccc776855..4276a2b97 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -275,7 +275,7 @@ private bool TryGetHandoff([NotNullWhen(true)] out Handoff? handoff) internal bool RetireTargetWithoutLock(string reason) { if (!TryGetHandoff(out var handoff) || handoff.Target is not { } target || !ReferenceEquals(physical, target)) return false; - target.MarkRetiredForHandoff(); + if (!target.TryMarkRetiredForHandoff()) return false; // a retire under the lock got there first; its teardown follows handoff.Retired = true; handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); RetireConnection(target, reason + " (without the write lock: last resort)"); @@ -309,30 +309,32 @@ private enum Replacement } /// - /// Classifies a connection against a handoff, from the connection's own facts (rule c), doomed dominating. The - /// target, or a reconnect adopted as one, is doomed whatever else is true of it. Where the remote address - /// identifies the node and could be read, it decides: back on the retiring address is doomed - including a - /// connection dialled through a stale pin that still named that node - and anywhere else is the replacement. - /// Otherwise a connection dialled through this handoff's successor pin went where the server said, - /// which is not the node being retired. Where the address identifies nothing (a tunnel, a fixed front address) - /// only time tells: while the target is still being drained a different current connection cannot be judged; - /// once the target is gone, nothing else will arrive, so what is current is the replacement. What is left is - /// an address that could not be read yet, which is unknown and defers to a later pass (rule b) - unless there - /// is nothing to compare it against anyway, in which case the connection is taken at its word rather than - /// held to the deadline with nothing ever able to judge it. + /// Classifies a connection against a handoff, from the connection's own facts (rule c). The target, or a + /// reconnect adopted as one, is doomed whatever else is true of it. A connection dialled through this + /// handoff's successor pin went where the server said to go, and is the replacement even if it reports the + /// retiring address: a proxy restarted in place keeps its address, and the server naming it as the successor + /// is the authority on that. (A stale pin from an earlier handoff confers nothing, since it is matched against + /// this handoff's successor; and a handoff that names no successor drops any earlier pin.) Otherwise, where the + /// remote address identifies the node and could be read, it decides: back on the retiring address is doomed + /// and anywhere else is the replacement. Where the address identifies nothing (a tunnel, a fixed front + /// address, a successor at the same address) only time tells: while the target is still being drained a + /// different current connection cannot be judged; once the target is gone, nothing else will arrive, so what is + /// current is the replacement. What is left is an address that could not be read yet, which is unknown and + /// defers to a later pass (rule b) - unless there is nothing to compare it against anyway, in which case the + /// connection is taken at its word rather than held to the deadline with nothing ever able to judge it. /// private static Replacement Classify(Handoff handoff, PhysicalConnection connection) { if (connection.WasHandoffTarget) return Replacement.Doomed; + if (connection.HandoffTargetDialed is { } dialed && handoff.Successor is { } successor && dialed.Equals(successor)) + { + return Replacement.Yes; + } var retiring = handoff.RetiringEndPoint; if (handoff.AddressIdentifiesNode && retiring is not null && connection.TryGetRemoteEndPoint() is { } landedOn) { return landedOn.Equals(retiring) ? Replacement.Doomed : Replacement.Yes; } - if (connection.HandoffTargetDialed is { } dialed && handoff.Successor is { } successor && dialed.Equals(successor)) - { - return Replacement.Yes; - } if (!handoff.AddressIdentifiesNode) { return handoff.Phase == HandoffPhase.Holding ? Replacement.Unknown : Replacement.Yes; @@ -340,16 +342,24 @@ private static Replacement Classify(Handoff handoff, PhysicalConnection connecti return retiring is null ? Replacement.Yes : Replacement.Unknown; } - /// Whether the backlog still holds work a handoff diverted; the protection stands until it has been written (rule a). + /// + /// Whether the backlog still holds work a handoff diverted, including a message the processor has dequeued but + /// not yet written; the protection stands until it has been written (rule a). The in-hand message is read + /// before and after the queue, so a dequeue that lands between the two is seen either in the queue or in hand. + /// private bool AnyHeldQueued() { + if (_backlogInHand is { HeldForHandoff: true }) return true; foreach (var message in _backlog) { if (message.HeldForHandoff) return true; } - return false; + return _backlogInHand is { HeldForHandoff: true }; } + /// The message the backlog processor has taken off the queue and not yet finished writing; see . + private volatile Message? _backlogInHand; + /// /// Whether a connection failure belongs to a maintenance handoff in progress: the target (retired by the /// drain, closed by the server at the end of the window, or dropped by itself), a reconnect that landed back @@ -616,8 +626,10 @@ private static bool ShouldHoldBatch(List messages) // Under the lock only the marking: from here no writer that takes the lock writes to the connection (see // TryRequeueForHandoff), and the backlog processor skips it too, so the teardown itself can happen - // after the release rather than stalling every writer behind event handlers and socket shutdown. - toRetire.MarkRetiredForHandoff(); + // after the release rather than stalling every writer behind event handlers and socket shutdown. The + // marking is a compare-exchange, so of this and a lockless last-resort retire racing it, exactly one + // tears the connection down. + if (!toRetire.TryMarkRetiredForHandoff()) return handoff.Retired; handoff.Retired = true; handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); } @@ -1587,6 +1599,11 @@ internal void AcceptRerouted(Message message) /// spent counts) behind heads with newer ones, so where its own timeout rather than the shared deadline is the /// later of the two it can be reported late - by at most the heads' remaining budget - never early, and never /// lost. + /// + /// Internal traffic (keep-alives, probes) is not held: a stale probe has nothing to gain from lingering to the + /// handoff's deadline, and it is never diverted in the first place. An internal message behind held heads is + /// at worst reported late, which costs nothing. + /// /// /// The message to queue. /// The message was diverted by a handoff's hold (or requeued after the retire): held @@ -1595,7 +1612,10 @@ internal void AcceptRerouted(Message message) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void BacklogEnqueue(Message message, bool diverted = false) { - if (_handoff is { } handoff && (diverted || TryGetHandoff(out _))) message.MarkHeldForHandoff(handoff.Deadline.Ticks); + // the same handoff decides the gate and the stamp: a message gated on one handoff must not carry the + // deadline of the one before it (a diverted message after the hold resolved takes the last handoff's) + var live = TryGetHandoff(out var handoff); + if ((diverted || live) && !message.IsInternalCall && (handoff ?? _handoff) is { } stamp) message.MarkHeldForHandoff(stamp.Deadline.Ticks); bool wasEmpty = !BacklogHasItems; // important that this *precedes* enqueue, to play well with HasPendingCallerFacingItems Interlocked.Increment(ref _backlogCurrentEnqueued); @@ -1961,6 +1981,7 @@ private void ProcessBridgeBacklog() { break; } + _backlogInHand = message; // still "queued" as far as a handoff's flush is concerned, until written } try @@ -1986,6 +2007,7 @@ private void ProcessBridgeBacklog() } finally { + _backlogInHand = null; UnmarkActiveMessage(message); } } diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 1068fab6f..581b9e636 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -983,13 +983,16 @@ internal void GetStormLog(StringBuilder sb) /// reached the write lock afterwards can tell a planned replacement from an ordinary closure. Set under the /// bridge's write lock; the report and dispose follow once the lock is released. Never reset. /// - internal bool RetiredForHandoff => _retiredForHandoff; + internal bool RetiredForHandoff => Volatile.Read(ref _retiredForHandoff) != 0; - internal void MarkRetiredForHandoff() => _retiredForHandoff = true; + /// Marks the connection retired, once: only the caller that marked it tears it down. + internal bool TryMarkRetiredForHandoff() => Interlocked.CompareExchange(ref _retiredForHandoff, 1, 0) == 0; + + private int _retiredForHandoff; // volatile: each is written on one thread and read on others as part of the handoff's identity test, in the // very races that test exists for; a plain field could be read stale exactly then - private volatile bool _retiredForHandoff, _wasHandoffTarget, _everFullyEstablished; + private volatile bool _wasHandoffTarget, _everFullyEstablished; private volatile System.Net.EndPoint? _handoffTargetDialed; /// @@ -1128,6 +1131,12 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou } } } + else if (msg.HeldForHandoff) + { + // a command a handoff held carries a longer cover than the ordinary ones written behind it on + // the replacement, so it says nothing about them: look past it + continue; + } else { // This is a head-of-line queue, which means the first thing we hit that *hasn't* timed out means no more will timeout diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index a35dec4e5..434ed06ea 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -47,17 +47,14 @@ private MaintenanceNotificationMode MaintenanceMode : Multiplexer.RawConfig.MaintenanceNotifications; /// - /// Whether anything can change a command's timeout against this server while it waits, so that a sync waiter - /// knows whether to re-read its budget or take it in one wait. Relaxation and the handoff cover both come from - /// notifications, so: a server that has ever accepted the opt-in (the negotiation record, kept across handshakes - /// because the current connection's is cleared at the start of each one and a notification can arrive the moment - /// a handshake completes), a window already open (which can still close, shrinking the budget back), or a handoff - /// in flight. A server that refused, or was never asked, cannot change anything. + /// Whether a command's timeout against this server can shrink while a sync caller waits, so that the + /// caller knows whether to wake and re-read its budget in slices or take it in one wait: a relaxed window is + /// open (it can close, and the budget drops back to the configured value), or a handoff is in flight. An + /// extension needs no slicing - the budget is re-read after the wait either way, and a wait that ended under a + /// budget that has since grown simply continues - so a server with nothing in progress costs a sync caller + /// nothing, however long its calls. /// - internal bool MaintenanceMayChangeTimeouts - => _maintenanceNotificationsEverAccepted || GetRelaxedRemaining() > 0 || Volatile.Read(ref _handoffInFlight) != 0; - - private volatile bool _maintenanceNotificationsEverAccepted; + internal bool MaintenanceMayChangeTimeouts => GetRelaxedRemaining() > 0 || Volatile.Read(ref _handoffInFlight) != 0; /// /// Whether to ask this server for maintenance notifications during handshake. @@ -115,7 +112,7 @@ private RedisValue MaintenanceMovingEndpointTypeLiteral(PhysicalConnection conne internal void OnMaintenanceNotificationsAccepted(PhysicalConnection connection) { - _maintenanceNotificationsActive = _maintenanceNotificationsEverAccepted = true; + _maintenanceNotificationsActive = true; Multiplexer.Logger?.LogInformationMaintenanceNotificationsAccepted(new(this)); } @@ -501,7 +498,11 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining); - await DrainThenRecycleAsync(remaining, decision.Reason, target, replacementAddressSeen: true, currentEndPoint).ForAwait(); + // A successor at the very address being retired - a proxy restarted in place - means the address + // identifies nothing for this handoff: every connection, doomed or replacement, reports it. The + // pin-dialled test then governs, and after that only time. + var sameAddress = target is IPEndPoint successorAddress && currentEndPoint is not null && successorAddress.Equals(currentEndPoint); + await DrainThenRecycleAsync(remaining, decision.Reason, target, replacementAddressSeen: !sameAddress, currentEndPoint).ForAwait(); replaced = true; break; default: @@ -510,6 +511,10 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint break; } + // The one-at-a-time guard covers the decision and the drain, not the warning: that is observability, and + // holding the guard through its wait would drop a genuine second MOVING inside the first window outright - + // no pin, no hold - which is the pre-handoff failure for the second window. + Volatile.Write(ref _handoffInFlight, 0); if (replaced) await WarnIfNotEstablishedInTimeAsync(watch, window).ForAwait(); } catch (Exception ex) @@ -609,9 +614,10 @@ private TimeSpan DrainCap(TimeSpan budget) /// The successor the server named and reconnection is pinned to, if any: a connection /// dialled through it is the replacement. /// Whether the handoff has seen a second address - the server named a - /// successor, or DNS moved to a new address - so that a reconnect reporting the retiring address is known to be - /// back on the retiring node, and is retired in its turn; see . - /// Where no second address was ever involved, the address says nothing and only time tells. + /// successor at a different address, or DNS moved to a new address - so that a reconnect reporting the retiring + /// address is known to be back on the retiring node, and is retired in its turn; see + /// . Where no second address was ever involved (including a + /// successor at the same address), the address says nothing and only time tells. /// The address and port of the node being retired, as seen from the connection that /// was told, so that a replacement which landed elsewhere is recognised as such. private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, EndPoint? successor, bool replacementAddressSeen, IPEndPoint? retiringEndPoint) From c48c8e6fe104b7752ad263aff4333e70e49f2457 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 20:50:33 +0000 Subject: [PATCH 22/27] Handoff: the claim is released once, the pre-hold queue is covered, and judgement needs evidence Round 21. HandoffAsync releases its one-at-a-time claim exactly once, by whichever path still holds it, so a second handoff's claim is not stomped by the first's finally. BeginHold stamps the work already queued when the hold begins, since the hold delays it as it delays diverted work; BacklogEnqueue re-reads the handoff after the enqueue, so the two sides pair and a concurrent enqueue is stamped by one of them. Sync waiters re-evaluate per wait whether their budget can shrink. The flush's termination check takes the backlog lock, under which the processor moves a message from the queue to its hand. A connection with no positive evidence either way is unknown, never a replacement by default. The drain's polls settle the no-retire cases without the write lock. The relaxation ceiling is stated once, in ClampToRelaxationCeiling. --- .../ConnectionMultiplexer.cs | 4 +- src/StackExchange.Redis/PhysicalBridge.cs | 80 +++++++++++++++---- .../ServerEndPoint.Maintenance.cs | 36 +++++---- 3 files changed, 88 insertions(+), 32 deletions(-) diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index e0a76d333..fdc337cef 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2742,10 +2742,12 @@ internal static void ThrowFailed(TaskCompletionSource? source, Exception u // never see a relaxation, or a handoff's cover, that began after we parked. Only where the // budget can change at all, though: with maintenance notifications disabled for the server // nothing ever relaxes or covers it, and one wait for the whole budget is enough. - var sliceMilliseconds = server is not null && server.MaintenanceMayChangeTimeouts ? 1000 : int.MaxValue; var startedAt = Environment.TickCount; while (true) { + // per wait, not once: a window that opens after we park puts us on slices (so we notice it + // close again), and one that closes takes us off them + var sliceMilliseconds = server is not null && server.MaintenanceMayChangeTimeouts ? 1000 : int.MaxValue; var budget = server is null ? TimeoutMilliseconds : ServerEndPoint.GetEffectiveTimeoutMilliseconds(server.GetEffectiveTimeoutMilliseconds(TimeoutMilliseconds), message); diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 4276a2b97..ababd2890 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -319,9 +319,11 @@ private enum Replacement /// and anywhere else is the replacement. Where the address identifies nothing (a tunnel, a fixed front /// address, a successor at the same address) only time tells: while the target is still being drained a /// different current connection cannot be judged; once the target is gone, nothing else will arrive, so what is - /// current is the replacement. What is left is an address that could not be read yet, which is unknown and - /// defers to a later pass (rule b) - unless there is nothing to compare it against anyway, in which case the - /// connection is taken at its word rather than held to the deadline with nothing ever able to judge it. + /// current is the replacement. What is left - an address that could not be read yet, or nothing to compare it + /// against because the connection that was told had no readable address - is unknown and defers to a later + /// pass (rule b): a verdict of replacement needs positive evidence, never a default, since the alternative is + /// flushing held work onto a stale-DNS reconnect to the node being retired. The cost is that a connection + /// nothing can ever judge is held to the deadline. /// private static Replacement Classify(Handoff handoff, PhysicalConnection connection) { @@ -339,22 +341,26 @@ private static Replacement Classify(Handoff handoff, PhysicalConnection connecti { return handoff.Phase == HandoffPhase.Holding ? Replacement.Unknown : Replacement.Yes; } - return retiring is null ? Replacement.Yes : Replacement.Unknown; + return Replacement.Unknown; } /// /// Whether the backlog still holds work a handoff diverted, including a message the processor has dequeued but - /// not yet written; the protection stands until it has been written (rule a). The in-hand message is read - /// before and after the queue, so a dequeue that lands between the two is seen either in the queue or in hand. + /// not yet written; the protection stands until it has been written (rule a). Under the backlog lock, which is + /// where the processor moves a message from the queue to its hand, so the message is seen in one place or the + /// other. Only the termination path asks this, never the per-message path. /// private bool AnyHeldQueued() { - if (_backlogInHand is { HeldForHandoff: true }) return true; - foreach (var message in _backlog) + lock (_backlog) { - if (message.HeldForHandoff) return true; + if (_backlogInHand is { HeldForHandoff: true }) return true; + foreach (var message in _backlog) + { + if (message.HeldForHandoff) return true; + } + return false; } - return _backlogInHand is { HeldForHandoff: true }; } /// The message the backlog processor has taken off the queue and not yet finished writing; see . @@ -401,8 +407,21 @@ internal void BeginHold(TimeSpan protection, System.Net.IPEndPoint? retiringEndP // cannot complete, and its held work - which belongs to the bridge's backlog, not to a handoff - keeps its // stamped cover and is flushed with this handoff's. var previous = _handoff; - _handoff = new Handoff(target, retiringEndPoint, successor, addressIdentifiesNode, protection); + var handoff = new Handoff(target, retiringEndPoint, successor, addressIdentifiesNode, protection); + _handoff = handoff; previous?.Finish(); + + // Work already in the backlog when the hold begins is delayed by it exactly as diverted work is (the + // processor does not flush while the hold is unresolved), so it gets the same cover: without this, a + // queue that predates the MOVING would be timed out by the hold itself under short timeouts with + // relaxation off. Store the handoff, fence, then walk the queue - paired with BacklogEnqueue, which + // enqueues, fences, then re-reads the handoff - so a message enqueued concurrently is stamped by one side + // or the other. + Interlocked.MemoryBarrier(); + foreach (var queued in _backlog) + { + if (!queued.IsInternalCall) queued.MarkHeldForHandoff(handoff.Deadline.Ticks); + } } /// @@ -558,10 +577,29 @@ private static bool ShouldHoldBatch(List messages) if (!TryGetHandoff(out var handoff)) return false; if (handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return handoff.Retired; - // Lock-free first: the drain polls this every 20ms, and the in-flight question only needs the connection's - // own lock. The write lock is taken only for a pass that is likely to retire, where it has to be held - // across the check and the marking. - if (!force && handoff.Target is { } busy && ReferenceEquals(physical, busy) && busy.HasCallerMessagesAwaitingResponse()) return null; + // Lock-free first: the drain polls this every 20ms, and only a pass that is going to retire needs the write + // lock (held across the check and the marking). The in-flight question needs only the connection's own + // lock; and with the target gone, finding the bridge disconnected, a replacement current, or a connection + // not yet classifiable settles the pass without a retire, so without the lock. The locked pass below + // repeats the judgement on what is current then, which is the one that counts. + var current = physical; + if (handoff.Target is { } drainTarget && ReferenceEquals(current, drainTarget)) + { + if (!force && drainTarget.HasCallerMessagesAwaitingResponse()) return null; + } + else + { + handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); + if (current is null) return false; + switch (Classify(handoff, current)) + { + case Replacement.Yes: + if (current.EverFullyEstablished) OnReplacementEstablished(handoff); + return false; + case Replacement.Unknown: + return null; + } + } PhysicalConnection toRetire; bool gotLock; @@ -583,7 +621,7 @@ private static bool ShouldHoldBatch(List messages) try { if (handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return handoff.Retired; - var current = physical; + current = physical; if (handoff.Target is { } target && ReferenceEquals(current, target)) { toRetire = target; @@ -1615,13 +1653,21 @@ private void BacklogEnqueue(Message message, bool diverted = false) // the same handoff decides the gate and the stamp: a message gated on one handoff must not carry the // deadline of the one before it (a diverted message after the hold resolved takes the last handoff's) var live = TryGetHandoff(out var handoff); - if ((diverted || live) && !message.IsInternalCall && (handoff ?? _handoff) is { } stamp) message.MarkHeldForHandoff(stamp.Deadline.Ticks); + var holdable = !message.IsInternalCall; + if ((diverted || live) && holdable && (handoff ?? _handoff) is { } stamp) message.MarkHeldForHandoff(stamp.Deadline.Ticks); bool wasEmpty = !BacklogHasItems; // important that this *precedes* enqueue, to play well with HasPendingCallerFacingItems Interlocked.Increment(ref _backlogCurrentEnqueued); Interlocked.Increment(ref _backlogTotalEnqueued); _backlog.Enqueue(message); message.SetBacklogged(); + if (holdable && !live) + { + // enqueue, fence, re-read: a hold that began between the read above and the enqueue has walked the + // queue without this message, or is about to walk it with it; either way it ends up stamped + Interlocked.MemoryBarrier(); + if (TryGetHandoff(out var late)) message.MarkHeldForHandoff(late.Deadline.Ticks); + } if (wasEmpty) { diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 434ed06ea..4cbd5dcff 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -261,13 +261,12 @@ internal bool OnMaintenanceWindowOpened(MaintenanceNotificationType type, long? var config = Multiplexer.RawConfig; var floor = config.MaintenanceRelaxedTimeout; - var cap = config.MaintenanceRelaxedWindowMax; // no time, or a time at or below zero, means "act now" rather than "ignore this": a connection that // arrives mid-window is told what is left of it, and that can legitimately be negative var duration = time is { } value && value > TimeSpan.Zero ? value : floor; if (duration < floor) duration = floor; - if (duration > cap) duration = cap; + duration = ClampToRelaxationCeiling(duration, TimeSpan.Zero); Volatile.Write(ref _relaxedType, (int)type); ExtendRelaxedWindow(duration, $"{type} for {duration.TotalSeconds}s"); @@ -445,6 +444,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint { var watch = ValueStopwatch.StartNew(); var replaced = false; + var claimed = true; // the one-at-a-time claim taken by OnMovingAnnounced; released once, by whoever holds it try { // Spread the fleet, but only by a fraction of the window - see MaintenanceHandoff.GetJitter for why @@ -514,6 +514,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // The one-at-a-time guard covers the decision and the drain, not the warning: that is observability, and // holding the guard through its wait would drop a genuine second MOVING inside the first window outright - // no pin, no hold - which is the pre-handoff failure for the second window. + claimed = false; Volatile.Write(ref _handoffInFlight, 0); if (replaced) await WarnIfNotEstablishedInTimeAsync(watch, window).ForAwait(); } @@ -523,7 +524,9 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint } finally { - Volatile.Write(ref _handoffInFlight, 0); + // only if still ours: after the early release the claim may belong to a second handoff, which must not + // find its guard gone and a third MOVING racing it + if (claimed) Volatile.Write(ref _handoffInFlight, 0); } } @@ -562,12 +565,22 @@ private TimeSpan DrainCap(TimeSpan budget) // the relaxation the drain depends on is itself bounded by the user's ceiling: holding for longer than the // sweep will relax would pay the hold's latency and still fail the command. The floor still applies, as for a // relaxed timeout of zero: a sub-second ceiling says "do not relax", not "do not drain". - var ceiling = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; - if (ceiling < DrainFloor) ceiling = DrainFloor; - if (cap > ceiling) cap = ceiling; + cap = ClampToRelaxationCeiling(cap, DrainFloor); return cap > budget ? budget : cap; } + /// + /// The one statement of what bounds: a span of + /// relaxation, or of anything that depends on relaxation, never exceeds it - except that a span may carry its own + /// floor, below which the ceiling does not cut (the drain's floor, the cover's drain-plus-connect). + /// + private TimeSpan ClampToRelaxationCeiling(TimeSpan span, TimeSpan floor) + { + var ceiling = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; + if (ceiling < floor) ceiling = floor; + return span > ceiling ? ceiling : span; + } + /// The least the drain waits for in-flight commands, whatever the relaxed timeout: one round trip on any network, with room to spare. private static readonly TimeSpan DrainFloor = TimeSpan.FromSeconds(1); @@ -634,19 +647,14 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, EndPoin // of the window plus one connect attempt: a MoveTo pins the socket target to the named successor for the // rest of the window, and a successor that does not answer is tried until that expires before the endpoint // is resolved the ordinary way. - var relax = cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout; - if (relax > Multiplexer.RawConfig.MaintenanceRelaxedWindowMax) relax = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; // the user's ceiling applies here as on the notification path - ExtendRelaxedWindow(relax, "MOVING handoff"); - var drainSpan = cap + LockGrace + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); + ExtendRelaxedWindow(ClampToRelaxationCeiling(cap + Multiplexer.RawConfig.MaintenanceRelaxedTimeout, TimeSpan.Zero), "MOVING handoff"); + var connect = TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); var pinned = cap + LockGrace > budget ? cap + LockGrace : budget; - var cover = pinned + TimeSpan.FromMilliseconds(Multiplexer.RawConfig.ConnectTimeout); // the user's ceiling on relaxation bounds the cover too - a huge announced window must not floor the held // commands' timeouts for its whole length - but the cover always spans at least the drain itself and one // connect attempt, which is what the held commands need; it applies to them alone, so an application that // has turned relaxation off keeps its short timeouts for everything else - var ceiling = Multiplexer.RawConfig.MaintenanceRelaxedWindowMax; - if (ceiling < drainSpan) ceiling = drainSpan; - if (cover > ceiling) cover = ceiling; + var cover = ClampToRelaxationCeiling(pinned + connect, cap + LockGrace + connect); // snapshots: Dispose nulls the fields, and a handoff in progress must not fault on a server going away var interactiveBridge = interactive; From ec4f853ebe51e1266035223d842478ef8806893f Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 21:56:00 +0000 Subject: [PATCH 23/27] Handoff: doomed is doomed in every phase, and "a handoff exists" is the phase, not the drain's claim Round 22. A reconnect that lands back on the retiring node while the held work is flushing puts the hold back in force (Flushing to AwaitingReplacement, before the connection reads as established) and is retired; the heartbeat judges a Flushing handoff too. A tunnel no longer records a successor dial that never happened. The drain's one-at-a-time claim keeps serialising the decision and the drain, but everything that asks whether a handoff exists - the relaxed window's shorten guard, the sync waiters' slicing - now asks the bridges' phases (HandoffLive); and a MOVING arriving while a drain is in flight updates the successor pin instead of being dropped. A diverted message whose hold has gone by the time it is enqueued is ordinary work rather than stamped from a dead handoff; the post-enqueue re-read rides on the total counter's interlocked increment, so a bridge that never saw a handoff pays one null read. A finished handoff lets go of its target. The docs state the cover's floor. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 53 +++++++++++++------ src/StackExchange.Redis/PhysicalConnection.cs | 4 +- .../ServerEndPoint.Maintenance.cs | 39 +++++++++----- 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 8df500282..d92c79ba5 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. The commands the hold diverted keep a longer timeout until they are answered - the hold, the rest of the window and one connect attempt, bounded by `maintRelaxedWindowMax` - whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; every other command on that server keeps its configured timeout throughout, including under an application that has turned relaxation off. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. The commands the hold diverted keep a longer timeout until they are answered - the hold, the rest of the window and one connect attempt, bounded by `maintRelaxedWindowMax` but never less than the hold plus one connect attempt, since that much is what the held commands need - whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; every other command on that server keeps its configured timeout throughout, including under an application that has turned relaxation off. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index ababd2890..7989eb16e 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -235,12 +235,20 @@ public Handoff(PhysicalConnection? target, System.Net.IPEndPoint? retiringEndPoi public HandoffPhase Phase => (HandoffPhase)Volatile.Read(ref _phase); - /// Advances the phase, if it is still where the caller saw it. + /// Advances the phase, if it is still where the caller saw it. Done lets go of the target: a finished handoff must not root a retired connection. public bool TryAdvance(HandoffPhase from, HandoffPhase to) - => Interlocked.CompareExchange(ref _phase, (int)to, (int)from) == (int)from; + { + if (Interlocked.CompareExchange(ref _phase, (int)to, (int)from) != (int)from) return false; + if (to == HandoffPhase.Done) Volatile.Write(ref _target, null); + return true; + } /// Ends the handoff whatever its phase: the deadline passed, or the bridge was disposed. - public void Finish() => Volatile.Write(ref _phase, (int)HandoffPhase.Done); + public void Finish() + { + Volatile.Write(ref _phase, (int)HandoffPhase.Done); + Volatile.Write(ref _target, null); + } } private volatile Handoff? _handoff; @@ -445,7 +453,7 @@ internal void EndDrain() /// private void ReevaluateHandoff() { - if (!TryGetHandoff(out var handoff) || handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return; + if (!TryGetHandoff(out var handoff)) return; if (physical is { EverFullyEstablished: true, RetiredForHandoff: false } current) Judge(handoff, current); } @@ -456,7 +464,10 @@ private void ReevaluateHandoff() /// the lock or faulted. The retire takes the write lock, so it runs apart from the caller (an establishment /// callback, or the heartbeat), one at a time and idempotent under the lock: a heartbeat that finds the /// previous attempt still waiting does not pile on, and one that finds it failed tries again. While the target - /// is still being drained, the drain loop owns the judgement. + /// is still being drained, the drain loop owns the judgement. Doomed is doomed in every phase after that: a + /// replacement that dropped mid-flush and a reconnect that landed back on the retiring node puts the hold back + /// in force (Flushing to AwaitingReplacement) before it is retired - the phase changes what release means, not + /// what doomed means. /// private void Judge(Handoff handoff, PhysicalConnection connection) { @@ -465,7 +476,8 @@ private void Judge(Handoff handoff, PhysicalConnection connection) case Replacement.Yes: OnReplacementEstablished(handoff); break; - case Replacement.Doomed when handoff.Phase == HandoffPhase.AwaitingReplacement: + case Replacement.Doomed when handoff.Phase is HandoffPhase.AwaitingReplacement or HandoffPhase.Flushing: + handoff.TryAdvance(HandoffPhase.Flushing, HandoffPhase.AwaitingReplacement); if ((ReferenceEquals(connection, handoff.Target) || handoff.TryAdopt(connection)) && handoff.TryBeginRetire()) { _ = RetireTargetAsync(handoff); @@ -474,6 +486,9 @@ private void Judge(Handoff handoff, PhysicalConnection connection) } } + /// Whether a handoff is in progress on this bridge, in any phase but Done; see . + internal bool HandoffLive => TryGetHandoff(out _); + /// /// The retire outside the drain loop. Forced: the loop's own forced pass has come and gone by the time this /// runs, and nothing a caller is waiting on can be in flight on a reconnect that the hold has diverted caller @@ -1178,6 +1193,14 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) connection.ReportAttemptOutcome(isSuccess: true, ConnectionFailureType.None, null); connection.MarkFullyEstablished(); connection.SetIdle(); + // Before the state changes to established, not after: a doomed connection becoming current while the + // held work is flushing must put the hold back in force before the processor can see a connected bridge + // with no hold and write to it. The judgement proper (adopt, retire) follows below, once established. + if (TryGetHandoff(out var early) && early.Phase == HandoffPhase.Flushing && physical == connection + && Classify(early, connection) == Replacement.Doomed) + { + early.TryAdvance(HandoffPhase.Flushing, HandoffPhase.AwaitingReplacement); + } if (physical == connection && !isDisposed && ChangeState(State.ConnectedEstablishing, State.ConnectedEstablished)) { reportNextFailure = reconfigureNextFailure = true; @@ -1652,22 +1675,22 @@ private void BacklogEnqueue(Message message, bool diverted = false) { // the same handoff decides the gate and the stamp: a message gated on one handoff must not carry the // deadline of the one before it (a diverted message after the hold resolved takes the last handoff's) + // A diverted message whose hold has gone by the time it is enqueued is ordinary work: it is flushed with the + // rest, and a stamp from a finished handoff would mark it held with no cover behind it. var live = TryGetHandoff(out var handoff); var holdable = !message.IsInternalCall; - if ((diverted || live) && holdable && (handoff ?? _handoff) is { } stamp) message.MarkHeldForHandoff(stamp.Deadline.Ticks); + if (live && holdable) message.MarkHeldForHandoff(handoff!.Deadline.Ticks); // live: TryGetHandoff said so bool wasEmpty = !BacklogHasItems; // important that this *precedes* enqueue, to play well with HasPendingCallerFacingItems Interlocked.Increment(ref _backlogCurrentEnqueued); - Interlocked.Increment(ref _backlogTotalEnqueued); _backlog.Enqueue(message); message.SetBacklogged(); - if (holdable && !live) - { - // enqueue, fence, re-read: a hold that began between the read above and the enqueue has walked the - // queue without this message, or is about to walk it with it; either way it ends up stamped - Interlocked.MemoryBarrier(); - if (TryGetHandoff(out var late)) message.MarkHeldForHandoff(late.Deadline.Ticks); - } + // The total is counted after the enqueue, deliberately: an interlocked increment is a full fence, and the + // re-read below needs one after the enqueue's store. Enqueue, fence, re-read: a hold that began between + // the read above and the enqueue has walked the queue without this message, or is about to walk it with + // it; either way it ends up stamped. A bridge that has never seen a handoff pays one null read for this. + Interlocked.Increment(ref _backlogTotalEnqueued); + if (holdable && !live && _handoff is not null && TryGetHandoff(out var late)) message.MarkHeldForHandoff(late.Deadline.Ticks); if (wasEmpty) { diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 581b9e636..4e9a274d0 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -316,7 +316,9 @@ internal async Task BeginConnectAsync(ILogger? log) { Trace($"handoff: connecting to {Format.ToString(handoffTarget)} in place of {Format.ToString(endpoint)}"); connectTo = handoffTarget; - _handoffTargetDialed = handoffTarget; + // only where the socket below really dials it: a tunnel connects to the configured endpoint on its + // own terms, and recording a dial that never happened would be false evidence for the handoff + if (tunnel is null) _handoffTargetDialed = handoffTarget; } if (tunnel is not null) { diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 4cbd5dcff..e53bdd1b9 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -54,7 +54,15 @@ private MaintenanceNotificationMode MaintenanceMode /// budget that has since grown simply continues - so a server with nothing in progress costs a sync caller /// nothing, however long its calls. /// - internal bool MaintenanceMayChangeTimeouts => GetRelaxedRemaining() > 0 || Volatile.Read(ref _handoffInFlight) != 0; + internal bool MaintenanceMayChangeTimeouts => GetRelaxedRemaining() > 0 || HandoffLive; + + /// + /// Whether a handoff is in progress on either bridge, in any phase but done - from the hold beginning until the + /// held work has been written to the replacement or the deadline passed. This, not the drain's one-at-a-time + /// claim, is what "a handoff exists" means to anything outside the drain: the claim covers the decision and the + /// drain, which is a shorter span than the handoff, and the two had been confused for each other. + /// + internal bool HandoffLive => interactive?.HandoffLive == true || subscription?.HandoffLive == true; /// /// Whether to ask this server for maintenance notifications during handshake. @@ -310,11 +318,12 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? Volatile.Write(ref _relaxedType, (int)type); var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; - // While a handoff is in flight, the drain depends on the relaxation it extended: a completion for some other - // operation arriving mid-drain (a MIGRATED for a shard move) must not pull the deadline back under the drain's - // cap and have the sweep fail the very commands the drain is waiting for. So during a handoff a close can + // While a handoff is live - from the hold beginning until the held work has reached the replacement - the + // drain and the reconnect depend on the relaxation the handoff extended: a completion for some other + // operation arriving meanwhile (a MIGRATED for a shard move) must not pull the deadline back under it and + // have the sweep fail the very commands the handoff is carrying across. So during a handoff a close can // lengthen the window but not shorten it; the replace semantics above apply the rest of the time. - var handoffInFlight = Volatile.Read(ref _handoffInFlight) != 0; + var handoffInFlight = HandoffLive; if (tail <= TimeSpan.Zero) { if (!handoffInFlight) _relaxedDeadline.SetIn(TimeSpan.Zero); // ended now, and recorded as such for fault attribution @@ -422,19 +431,21 @@ internal void SetHandoffTarget(EndPoint target, TimeSpan window) /// internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalConnection connection) { - // One at a time per server. A rolling operation delivers one MOVING per connection, so a second one - // arriving while a handoff is in flight is a repeat or a much later event; either way, starting a - // second poll against the same endpoint achieves nothing. - if (Interlocked.CompareExchange(ref _handoffInFlight, 1, 0) != 0) - { - Multiplexer.Trace("MOVING: a handoff is already in flight", ToString()); - return; - } - var budget = window is { } value && value > TimeSpan.Zero ? value : Multiplexer.RawConfig.MaintenanceRelaxedTimeout; // no window given: use the relaxation floor if (budget > MaxHandoffWindow) budget = MaxHandoffWindow; // an absurd announced window must not overflow a delay or a deadline + + // One decision-and-drain at a time per server: two drains on the same bridges would end each other's holds. + // A MOVING arriving while one is in flight (which can be most of a window: the jitter, the DNS poll, the + // drain) is not dropped, though - the handoff in flight will replace the connections, and what this one adds + // is where the reconnect should go, if it says: the pin is updated, and the next dial takes it. + if (Interlocked.CompareExchange(ref _handoffInFlight, 1, 0) != 0) + { + if (successor is not null) SetHandoffTarget(successor, budget); + Multiplexer.Trace(successor is null ? "MOVING: a handoff is already in flight" : "MOVING: a handoff is already in flight; successor noted", ToString()); + return; + } var current = connection.TryGetRemoteEndPoint(); // null if the socket is gone or not connected: nothing to compare against _ = HandoffAsync(budget, successor, current); From 21ed775e2cda4ce915299eeb3c362e76a64b94e5 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 5 Nov 2026 22:33:53 +0000 Subject: [PATCH 24/27] Handoff: the cover follows the write time into transactions, and the pin is one generation-checked state Round 23. InheritWriteTime carries the parent's hold cover as well as its write time, so a held transaction's inner commands are not failed by the sweep on the replacement for a clock that includes the hold. An internal member of a held batch is covered with the batch, since the batch is held as a unit and the sweep must not fail one member of it. The pin - target, expiry, and the generation of the MOVING that set it - is kept consistent under the maintenance lock: a handoff's late decision cannot clear or overwrite a pin a later MOVING set, and the heartbeat's expiry check cannot null a pin set a moment ago. BacklogEnqueue loses its dead parameter and its stale contract. The docs and the drain comment say what a sub-second window does with the hold (it begins, for the instant of the replacement, because the retire is built on it). The drain-cap test's bound sits between the mechanism's worst case and the deadline it guards against. --- docs/ServerMaintenanceEvent.md | 2 +- src/StackExchange.Redis/Message.cs | 8 ++- src/StackExchange.Redis/PhysicalBridge.cs | 29 ++++---- .../ServerEndPoint.Maintenance.cs | 71 ++++++++++++------- .../MaintenanceNotificationTests.cs | 10 +-- 5 files changed, 78 insertions(+), 42 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index d92c79ba5..82a43cd0b 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -115,7 +115,7 @@ or `ServerDefault` to ask for nothing at all, which is what earlier versions did So an application that does nothing at all still benefits: commands that would have timed out during a migration are given more room, a moved slot is learned without waiting to be redirected, and a `MOVING` is acted on before the server closes the socket. -During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no hold at all, and the connections are replaced at once. The commands the hold diverted keep a longer timeout until they are answered - the hold, the rest of the window and one connect attempt, bounded by `maintRelaxedWindowMax` but never less than the hold plus one connect attempt, since that much is what the held commands need - whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; every other command on that server keeps its configured timeout throughout, including under an application that has turned relaxation off. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no drain, and the connections are replaced at once (commands are held only for the instant of the replacement itself). The commands the hold diverted keep a longer timeout until they are answered - the hold, the rest of the window and one connect attempt, bounded by `maintRelaxedWindowMax` but never less than the hold plus one connect attempt, since that much is what the held commands need - whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; every other command on that server keeps its configured timeout throughout, including under an application that has turned relaxation off. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. ### Notifications that arrive as you connect diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index f63d52199..bcdbc6acb 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -892,7 +892,13 @@ internal void SetWriteTime() /// ; they take the parent's write time, so that they share its timeout budget /// (including any time it spent in the backlog) rather than looking overdue by the process uptime. /// - internal void InheritWriteTime(Message parent) => _writeTickCount = parent._writeTickCount; + internal void InheritWriteTime(Message parent) + { + _writeTickCount = parent._writeTickCount; + // and the cover that goes with that budget: a held transaction's inner commands reach the replacement with + // a clock that includes the whole hold, and without the parent's cover the next sweep would fail them + _holdCover.ExtendTo(parent._holdCover.Ticks); + } /// /// The message is going back to the backlog after the connection it was headed for was retired by a maintenance diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 7989eb16e..7aeac445c 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -1662,21 +1662,20 @@ internal void AcceptRerouted(Message message) /// lost. /// /// Internal traffic (keep-alives, probes) is not held: a stale probe has nothing to gain from lingering to the - /// handoff's deadline, and it is never diverted in the first place. An internal message behind held heads is - /// at worst reported late, which costs nothing. + /// handoff's deadline, and it is never diverted on its own. An internal message behind held heads is at worst + /// reported late, which costs nothing. The exception is an internal member of a held batch, which the batch + /// helper stamps itself: the batch is held as a unit, and the sweep must not fail one member of it. + /// + /// + /// A message diverted by the hold whose handoff has gone by the time it is enqueued is ordinary work: it is + /// flushed with the rest, and a stamp from a finished handoff would mark it held with no cover behind it. /// /// - /// The message to queue. - /// The message was diverted by a handoff's hold (or requeued after the retire): held - /// whatever the phase is by now, since a hold that resolved between the gate and the enqueue does not make it - /// ordinary work. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void BacklogEnqueue(Message message, bool diverted = false) + private void BacklogEnqueue(Message message) { // the same handoff decides the gate and the stamp: a message gated on one handoff must not carry the // deadline of the one before it (a diverted message after the hold resolved takes the last handoff's) - // A diverted message whose hold has gone by the time it is enqueued is ordinary work: it is flushed with the - // rest, and a stamp from a finished handoff would mark it held with no cover behind it. var live = TryGetHandoff(out var handoff); var holdable = !message.IsInternalCall; if (live && holdable) message.MarkHeldForHandoff(handoff!.Deadline.Ticks); // live: TryGetHandoff said so @@ -1745,17 +1744,23 @@ private void EnqueueHeldAndWakeIfReleased(Message message, bool requeue = false) { if (requeue) message.SetRequeued(); else message.SetEnqueued(null); - BacklogEnqueue(message, diverted: true); + BacklogEnqueue(message); WakeIfHoldReleased(); } - /// A batch held as a unit: every member queued, then one fence and one check for the batch. + /// + /// A batch held as a unit: every member queued, then one fence and one check for the batch. An internal member + /// rides along with the rest (see ), so it is covered with the rest: left to the + /// ordinary timeout it would be failed by the sweep while the batch waits, rather than merely reported late. + /// private void EnqueueHeldBatchAndWakeIfReleased(List messages) { + var cover = TryGetHandoff(out var handoff) ? handoff.Deadline.Ticks : 0; foreach (var message in messages) { message.SetEnqueued(null); - BacklogEnqueue(message, diverted: true); + if (message.IsInternalCall) message.MarkHeldForHandoff(cover); // zero (no live handoff) covers nothing + BacklogEnqueue(message); } WakeIfHoldReleased(); } diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index e53bdd1b9..63a59926b 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -336,8 +336,13 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } - private volatile EndPoint? _handoffTarget; + // The pin: the target, its expiry and the generation of the MOVING that set it, kept consistent under + // _maintenanceSync (every touch is rare: a MOVING, a connect, a heartbeat), which is what lets the heartbeat's + // expiry check and a fresh SetHandoffTarget coexist without one nulling the other's pin. + private EndPoint? _handoffTarget; private TickDeadline _handoffTargetExpiry; + private int _handoffTargetGeneration; + private int _movingGeneration; // one per MOVING acted on; a later MOVING's pin outranks an earlier one's decision private int _handoffInFlight, _handoffRecycles; /// @@ -363,33 +368,48 @@ internal EndPoint? HandoffTarget { get { - var target = _handoffTarget; - if (target is null) return null; - - if (!_handoffTargetExpiry.IsActive) + lock (_maintenanceSync) { - _handoffTarget = null; // expired; resolve the endpoint the usual way from here on - return null; + if (_handoffTarget is null) return null; + if (!_handoffTargetExpiry.IsActive) + { + _handoffTarget = null; // expired; resolve the endpoint the usual way from here on + return null; + } + return _handoffTarget; } - - return target; } } /// - /// Points the next connection attempt at a named replacement, for as long as the announced window lasts. + /// Points the next connection attempt at a named replacement, for as long as the announced window lasts - unless + /// a later MOVING has pinned something since (a handoff's decision can resolve, after a DNS poll, well + /// after the next notification arrived; the later one knows more). /// - internal void SetHandoffTarget(EndPoint target, TimeSpan window) + private void SetHandoffTarget(EndPoint target, TimeSpan window, int generation) { - _handoffTargetExpiry.SetIn(window < TimeSpan.FromSeconds(1) ? TimeSpan.FromSeconds(1) : window); - _handoffTarget = target; + lock (_maintenanceSync) + { + if (_handoffTarget is not null && generation < _handoffTargetGeneration) return; + _handoffTargetExpiry.SetIn(window < TimeSpan.FromSeconds(1) ? TimeSpan.FromSeconds(1) : window); + _handoffTarget = target; + _handoffTargetGeneration = generation; + } } /// - /// Drops the pin when a later MOVING names no successor: the pinned node is then, as likely as not, the - /// very one now being retired, and a reconnect dialled through the stale pin would land on it. + /// Drops the pin when a MOVING names no successor: the pinned node is then, as likely as not, the very one + /// now being retired, and a reconnect dialled through the stale pin would land on it. Only a pin this or an earlier + /// MOVING set: a later one's stands. /// - private void ClearHandoffTarget() => _handoffTarget = null; + private void ClearHandoffTarget(int generation) + { + lock (_maintenanceSync) + { + if (_handoffTarget is not null && generation < _handoffTargetGeneration) return; + _handoffTarget = null; + } + } /// /// Reads the pin on every heartbeat so that its expiry is noticed while it is fresh. Expiry is lazy, on read, @@ -440,18 +460,19 @@ internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalC // A MOVING arriving while one is in flight (which can be most of a window: the jitter, the DNS poll, the // drain) is not dropped, though - the handoff in flight will replace the connections, and what this one adds // is where the reconnect should go, if it says: the pin is updated, and the next dial takes it. + var generation = Interlocked.Increment(ref _movingGeneration); if (Interlocked.CompareExchange(ref _handoffInFlight, 1, 0) != 0) { - if (successor is not null) SetHandoffTarget(successor, budget); + if (successor is not null) SetHandoffTarget(successor, budget, generation); Multiplexer.Trace(successor is null ? "MOVING: a handoff is already in flight" : "MOVING: a handoff is already in flight; successor noted", ToString()); return; } var current = connection.TryGetRemoteEndPoint(); // null if the socket is gone or not connected: nothing to compare against - _ = HandoffAsync(budget, successor, current); + _ = HandoffAsync(budget, successor, current, generation); } - private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint? currentEndPoint) + private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint? currentEndPoint, int generation) { var watch = ValueStopwatch.StartNew(); var replaced = false; @@ -489,7 +510,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint case HandoffAction.Recycle: // DNS has moved to a new address, so a reconnect reporting the old one is back on the retiring // node; a pin left by an earlier handoff is stale now and must not dial it - ClearHandoffTarget(); + ClearHandoffTarget(generation); await DrainThenRecycleAsync(remaining, decision.Reason, successor: null, replacementAddressSeen: true, currentEndPoint).ForAwait(); replaced = true; break; @@ -499,7 +520,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint var half = TimeSpan.FromTicks(window.Ticks / 2) - jitter; if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); // nothing to re-resolve, or nothing known to compare against: no second address has been seen - ClearHandoffTarget(); + ClearHandoffTarget(generation); await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successor: null, replacementAddressSeen: false, currentEndPoint).ForAwait(); replaced = true; break; @@ -508,7 +529,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // this only re-read the topology and recycled, which measurably did not work: we recycled // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. - SetHandoffTarget(target, remaining); + SetHandoffTarget(target, remaining, generation); // A successor at the very address being retired - a proxy restarted in place - means the address // identifies nothing for this handoff: every connection, doomed or replacement, reports it. The // pin-dialled test then governs, and after that only time. @@ -647,9 +668,11 @@ private TimeSpan ClampToRelaxationCeiling(TimeSpan span, TimeSpan floor) private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, EndPoint? successor, bool replacementAddressSeen, IPEndPoint? retiringEndPoint) { var watch = ValueStopwatch.StartNew(); - // With less than the floor left of the window there is no hold worth having: the server is about to close + // With less than the floor left of the window there is no drain worth having: the server is about to close // the socket regardless, so the connections are replaced at once (still under the lock) and the replacement - // gets what little time remains. Otherwise the hold lasts up to the cap. + // gets what little time remains. The hold still begins, because the retire is built on it - the target, the + // requeue of a writer that reaches the lock late, the cover for what the reconnect carries - but it lasts + // only the instant of the replacement. Otherwise the hold lasts up to the cap. var cap = budget < DrainFloor ? TimeSpan.Zero : DrainCap(budget); // The held work, and any blocking command being waited for, must not be timed out by the hold itself, nor diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 460c7a6dd..3b8ad6246 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -1017,7 +1017,7 @@ public async Task MovingWithBlockingCommandsInFlightRecyclesAtTheDrainCap() Assert.Equal(MaintenanceNotificationType.Moving, moving.NotificationType); Assert.True( - await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 12_000), + await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 14_000), "the handoff should have replaced the connection"); var recycledAfter = watch.Elapsed; // Only now: the server reads its latency when it *executes* a request, and zeroing it any earlier races the @@ -1026,9 +1026,11 @@ await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 1 server.SetLatency(TimeSpan.Zero); log.WriteLine($"recycled after {recycledAfter.TotalMilliseconds:0}ms with a 12s command in flight"); - // jitter (at most a tenth of the window, capped at 1s) + the 5s cap + the lock grace (1s) + poll granularity, - // with a further second of slack: still well short of the 12s the command needs - Assert.True(recycledAfter < TimeSpan.FromSeconds(10), $"recycled after {recycledAfter}: the drain waited for the blocking command instead of capping"); + // jitter (at most a tenth of the window, capped at 1s) + the 5s cap + the lock grace (1s) + poll granularity + // is the mechanism's own worst case, about 7.5s; the bug being guarded against is a recycle at the 15s + // deadline, so 12s discriminates with room on both sides, where 10s left two to three seconds over the + // worst case on a slow machine + Assert.True(recycledAfter < TimeSpan.FromSeconds(12), $"recycled after {recycledAfter}: the drain waited for the blocking command instead of capping"); // the in-flight command was failed by the recycle, which is what the server's close would have done foreach (var pending in blocking) From db4c0ad2e002b0f06cdac5446626cab30540b90b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 6 Nov 2026 09:12:58 +0000 Subject: [PATCH 25/27] Handoff: the replacement is whatever establishes once the target is gone; the contract; a two-MOVINGs matrix Round 24, and the focusing note that followed it. The address-identity apparatus goes: the retiring address, the per-bridge comparison, the tunnel and front-address cases, the same-address successor, the pin-dialled fact, the adoption and re-retire of a doomed reconnect. The replacement is the connection that fully establishes once the target is gone, and nothing else; a reconnect back on the retiring node, with no successor named and a stale resolver, is closed by the server at the end of the window and reconnected again, as it always was, with the relaxed window and the held work's cover meanwhile. Eight review rounds of findings were in that code, for that one corner. The pin's generation check is unconditional, a successor-less MOVING mid-drain drops the pin, and the fault-attribution type follows the window it describes. The contract is written into the docs: one handoff at a time per endpoint is fully covered; back-to-back MOVINGs in one window are best effort - nothing lost or abandoned contrary to policy, a bounded wait, resolved later rather than wrong. A six-case test matrix (second MOVING with or without a successor, arriving during the first handoff's decision, drain or flush) asserts exactly that and nothing about the mechanism. --- docs/ServerMaintenanceEvent.md | 2 + src/StackExchange.Redis/PhysicalBridge.cs | 233 +++++------------- src/StackExchange.Redis/PhysicalConnection.cs | 16 +- .../ServerEndPoint.Maintenance.cs | 61 +++-- .../MaintenanceNotificationTests.cs | 106 ++++++++ 5 files changed, 198 insertions(+), 220 deletions(-) diff --git a/docs/ServerMaintenanceEvent.md b/docs/ServerMaintenanceEvent.md index 82a43cd0b..64c7af3da 100644 --- a/docs/ServerMaintenanceEvent.md +++ b/docs/ServerMaintenanceEvent.md @@ -117,6 +117,8 @@ So an application that does nothing at all still benefits: commands that would h During that handoff, commands the application issues are not written to the connection being retired: they wait in the backlog and are sent on the replacement once it is established, approximately in the order they were issued (a command that was racing the replacement itself may be reordered against the ones issued just after it). While the old connection is being drained this happens whatever the `backlogPolicy`, because that connection is live and the command would otherwise have been written to it; and a `FailFast` policy does not abandon the held work when the old connection is closed, or when an attempt to reach the replacement fails, since a handoff is a replacement the client chose rather than a failure. Between the old connection being replaced and the replacement completing its handshake, a `FailFast` policy fails *new* commands immediately, as it does on any reconnect; the held ones wait for the replacement. The hold lasts until the commands already in flight have been answered, and at most half of `maintRelaxedTimeout` (five seconds by default), never more than half of what is left of the announced window (a generous relaxed timeout must not become a full-window stall), never less than a second (which wins over the half-window bound when less than two seconds remain), and never longer than the window itself, since the server closes the socket when that ends regardless; with less than a second of window left there is no drain, and the connections are replaced at once (commands are held only for the instant of the replacement itself). The commands the hold diverted keep a longer timeout until they are answered - the hold, the rest of the window and one connect attempt, bounded by `maintRelaxedWindowMax` but never less than the hold plus one connect attempt, since that much is what the held commands need - whatever `maintRelaxedTimeout` is set to, so nothing is timed out by the hold itself; every other command on that server keeps its configured timeout throughout, including under an application that has turned relaxation off. A blocking command that was already in flight when the hold began and whose timeout fits inside the hold completes normally; one issued during the hold is held like everything else and runs on the replacement. A command still in flight when the hold runs out is failed by the replacement, as the server's close would have failed it, so that one blocking read cannot stall everything else for the rest of the window. +**What is guaranteed, and what is best effort.** One handoff at a time per endpoint is fully covered: commands already in flight drain (up to the cap above), commands issued during the hold are never failed by the hold itself, and they are written to the replacement or fail within the bound above (the hold, the rest of the window and one connect attempt). The replacement is whatever connection fully establishes once the old one has been retired; the client does not try to tell from a socket's address which node it reached, so a reconnect that lands back on the retiring node (no successor named, a stale resolver) is treated as the replacement and is closed by the server at the end of the window as before, with the relaxed window covering the gap. Back-to-back `MOVING`s inside one window are handled on a best-effort basis: a second notification while a handoff is in progress updates where the reconnect goes (or drops a successor that may itself be retiring) rather than starting a second drain, held commands are never lost or abandoned contrary to the `backlogPolicy`, but they may wait longer than the single-handoff bound, and the client resolves the hold conservatively, later rather than wrong. + ### Notifications that arrive as you connect Redis Enterprise **retains the most recent completion** - `MIGRATED` or `FAILED_OVER` - and replays it to each connection that opts in, so a client that connects after a disruption still learns that it happened. Measured behaviour, worth knowing if you handle these events yourself: diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 7aeac445c..6b68aec1a 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -166,9 +166,9 @@ private enum HandoffPhase } /// - /// One maintenance handoff on this bridge: its phase, and the facts fixed when it began. A new object per - /// handoff, so that a transition requested against an earlier handoff (a backlog pass finishing an earlier - /// flush, say) cannot land on a later one. + /// One maintenance handoff on this bridge: its phase, its target, and its deadline. A new object per handoff, + /// so that a transition requested against an earlier handoff (a backlog pass finishing an earlier flush, say) + /// cannot land on a later one. /// private sealed class Handoff { @@ -185,46 +185,15 @@ private sealed class Handoff /// public Maintenance.TickDeadline Deadline; - /// The connection the hold began on, or a reconnect adopted in its place; null if the bridge had none. + /// The connection the hold began on; null if the bridge had none. Let go of once the handoff is done. public PhysicalConnection? Target => Volatile.Read(ref _target); - /// - /// Adopts a reconnect that landed back on the node being retired as the target, once: of two callers that - /// see the same doomed connection (its establishment and a heartbeat, say) one adopts and retires it. - /// - public bool TryAdopt(PhysicalConnection doomed) - { - var previous = Volatile.Read(ref _target); - if (ReferenceEquals(previous, doomed) || Interlocked.CompareExchange(ref _target, doomed, previous) != previous) return false; - doomed.MarkHandoffTarget(); - return true; - } - - /// The successor this handoff pinned reconnection to, if the server named one; a connection dialled through it is the replacement. - public readonly System.Net.EndPoint? Successor; - /// Whether this handoff retired a connection (rather than finding it gone). public volatile bool Retired; - /// The address and port of the node being retired, as seen from the connection that was told; null if unreadable. - public readonly System.Net.IPEndPoint? RetiringEndPoint; - - /// - /// Whether a socket's remote address says which node it reached. Only when the handoff has itself seen a - /// second address - the server named a successor, or DNS moved to a new address - and there is no tunnel - /// in the way: then a reconnect reporting the retiring address is back on the retiring node. Behind a - /// tunnel, or when the only address ever involved is the one in front of the cluster, every connection - /// reports the same address and comparing it would brand the genuine replacement doomed forever; then - /// only time tells. - /// - public readonly bool AddressIdentifiesNode; - - public Handoff(PhysicalConnection? target, System.Net.IPEndPoint? retiringEndPoint, System.Net.EndPoint? successor, bool addressIdentifiesNode, TimeSpan protection) + public Handoff(PhysicalConnection? target, TimeSpan protection) { _target = target; - RetiringEndPoint = retiringEndPoint; - Successor = successor; - AddressIdentifiesNode = addressIdentifiesNode; Deadline.SetIn(protection); } @@ -294,62 +263,37 @@ internal bool RetireTargetWithoutLock(string reason) /// The rules the handoff keeps, stated once so that the next edit is checked against them rather than /// against the last edit: /// - /// (a) protection stands down only when the held work is safe - written to a bona fide - /// replacement, or past the deadline that also bounds its cover - never merely because a replacement - /// exists; - /// (b) classifying a connection has three outcomes, replacement, doomed or unknown, and unknown always - /// defers to a later pass - it is never acted on as either of the others; - /// (c) what a connection is - the target, made under the successor pin, established - is - /// recorded on the connection when it happens and read from there; the handoff's phase says only where the - /// flow stands, and nothing about a connection is inferred from it. + /// (a) protection stands down only when the held work is safe - written to the replacement, + /// or past the deadline that also bounds its cover - never merely because a replacement exists; + /// (b) the replacement is whatever fully establishes once the target is gone, and nothing else: the + /// client does not try to tell from a socket's address which node it reached. Address evidence (is this + /// reconnect back on the node being retired?) was tried, and cost eight review rounds of special cases - a + /// tunnel, a fixed front address, a successor at the same address, a node with several addresses, an address + /// that could not be read - for one corner it would have served: a reconnect to the retiring node after the + /// retire, with no successor named and a stale resolver. There the server closes that reconnect at the end of + /// the window and the client reconnects again, as it always did; the held work rides the relaxed window and + /// its own cover meanwhile. Less machinery, resolved later rather than wrong; + /// (c) what a connection is - the target, established - is recorded on the connection when it + /// happens and read from there; the handoff's phase says only where the flow stands. /// /// private enum Replacement { - /// A bona fide replacement: not the target, and demonstrably not back on the node being retired. + /// Established once the target was gone: the replacement. Yes, - /// The target, or a reconnect that landed back on the node being retired: the server will close it. + /// The target: the server will close it. Doomed, - /// Not classifiable yet (its address cannot be read, or only time can tell and it has not yet): defer. + /// A different connection current while the target is still being drained: judged again once the drain has let go. Unknown, } - /// - /// Classifies a connection against a handoff, from the connection's own facts (rule c). The target, or a - /// reconnect adopted as one, is doomed whatever else is true of it. A connection dialled through this - /// handoff's successor pin went where the server said to go, and is the replacement even if it reports the - /// retiring address: a proxy restarted in place keeps its address, and the server naming it as the successor - /// is the authority on that. (A stale pin from an earlier handoff confers nothing, since it is matched against - /// this handoff's successor; and a handoff that names no successor drops any earlier pin.) Otherwise, where the - /// remote address identifies the node and could be read, it decides: back on the retiring address is doomed - /// and anywhere else is the replacement. Where the address identifies nothing (a tunnel, a fixed front - /// address, a successor at the same address) only time tells: while the target is still being drained a - /// different current connection cannot be judged; once the target is gone, nothing else will arrive, so what is - /// current is the replacement. What is left - an address that could not be read yet, or nothing to compare it - /// against because the connection that was told had no readable address - is unknown and defers to a later - /// pass (rule b): a verdict of replacement needs positive evidence, never a default, since the alternative is - /// flushing held work onto a stale-DNS reconnect to the node being retired. The cost is that a connection - /// nothing can ever judge is held to the deadline. - /// + /// Classifies a connection against a handoff; see rule (b). private static Replacement Classify(Handoff handoff, PhysicalConnection connection) { if (connection.WasHandoffTarget) return Replacement.Doomed; - if (connection.HandoffTargetDialed is { } dialed && handoff.Successor is { } successor && dialed.Equals(successor)) - { - return Replacement.Yes; - } - var retiring = handoff.RetiringEndPoint; - if (handoff.AddressIdentifiesNode && retiring is not null && connection.TryGetRemoteEndPoint() is { } landedOn) - { - return landedOn.Equals(retiring) ? Replacement.Doomed : Replacement.Yes; - } - if (!handoff.AddressIdentifiesNode) - { - return handoff.Phase == HandoffPhase.Holding ? Replacement.Unknown : Replacement.Yes; - } - return Replacement.Unknown; + return handoff.Phase == HandoffPhase.Holding ? Replacement.Unknown : Replacement.Yes; } /// @@ -398,13 +342,8 @@ private bool IsHandoffOwnFailure(PhysicalConnection connection) /// was in flight at that instant. /// /// For how long at most the handoff stands - the FailFast shield here and the timeout - /// cover on the held commands - if no bona fide replacement establishes and flushes first. - /// The address and port of the node being retired, as seen from the connection - /// that was told, if known; see . - /// The successor the server named and reconnection is pinned to, if any. - /// Whether a socket's remote address identifies the node it reached; see - /// . - internal void BeginHold(TimeSpan protection, System.Net.IPEndPoint? retiringEndPoint, System.Net.EndPoint? successor, bool addressIdentifiesNode) + /// cover on the held commands - if no replacement establishes and flushes first. + internal void BeginHold(TimeSpan protection) { var target = physical; target?.MarkHandoffTarget(); @@ -415,7 +354,7 @@ internal void BeginHold(TimeSpan protection, System.Net.IPEndPoint? retiringEndP // cannot complete, and its held work - which belongs to the bridge's backlog, not to a handoff - keeps its // stamped cover and is flushed with this handoff's. var previous = _handoff; - var handoff = new Handoff(target, retiringEndPoint, successor, addressIdentifiesNode, protection); + var handoff = new Handoff(target, protection); _handoff = handoff; previous?.Finish(); @@ -458,16 +397,11 @@ private void ReevaluateHandoff() } /// - /// An established connection, judged: a bona fide replacement resolves the hold; a doomed one, once the drain - /// loop is over, is retired - a reconnect that landed back on the node being retired (adopted first, as the - /// drain loop would have adopted it), or the target itself, still current because the loop's retire never got - /// the lock or faulted. The retire takes the write lock, so it runs apart from the caller (an establishment - /// callback, or the heartbeat), one at a time and idempotent under the lock: a heartbeat that finds the - /// previous attempt still waiting does not pile on, and one that finds it failed tries again. While the target - /// is still being drained, the drain loop owns the judgement. Doomed is doomed in every phase after that: a - /// replacement that dropped mid-flush and a reconnect that landed back on the retiring node puts the hold back - /// in force (Flushing to AwaitingReplacement) before it is retired - the phase changes what release means, not - /// what doomed means. + /// An established connection, judged: the replacement resolves the hold; the target, still current once the + /// drain loop is over (its retire never got the lock, or faulted), is retired again - one attempt at a time, + /// idempotent under the lock, so a heartbeat that finds the previous attempt still waiting does not pile on + /// and one that finds it failed tries again. While the target is still being drained, the drain loop owns + /// the judgement. /// private void Judge(Handoff handoff, PhysicalConnection connection) { @@ -476,12 +410,8 @@ private void Judge(Handoff handoff, PhysicalConnection connection) case Replacement.Yes: OnReplacementEstablished(handoff); break; - case Replacement.Doomed when handoff.Phase is HandoffPhase.AwaitingReplacement or HandoffPhase.Flushing: - handoff.TryAdvance(HandoffPhase.Flushing, HandoffPhase.AwaitingReplacement); - if ((ReferenceEquals(connection, handoff.Target) || handoff.TryAdopt(connection)) && handoff.TryBeginRetire()) - { - _ = RetireTargetAsync(handoff); - } + case Replacement.Doomed when handoff.Phase != HandoffPhase.Holding && ReferenceEquals(connection, handoff.Target): + if (handoff.TryBeginRetire()) _ = RetireTargetAsync(handoff); break; } } @@ -573,20 +503,18 @@ private static bool ShouldHoldBatch(List messages) /// /// Retires the connection the hold began on, under the write lock, once nothing a caller is waiting on is in /// flight on it - or regardless, when says the drain has waited long enough. If that - /// connection has gone by itself and what is current is a reconnect that landed back on the node being retired - /// (no successor pinned, DNS still naming the old node), that reconnect takes its place and is retired instead; - /// a bona fide replacement is left alone. The lock is what keeps a writer that passed the hold check before it - /// was set from slipping onto the old connection between the check and the retire; such a writer finds the - /// connection retired when it gets the lock, and queues for the replacement instead (see - /// ). + /// connection has gone by itself, whatever is current is the replacement (rule b) and is left alone. The lock + /// is what keeps a writer that passed the hold check before it was set from slipping onto the old connection + /// between the check and the retire; such a writer finds the connection retired when it gets the lock, and + /// queues for the replacement instead (see ). /// /// Replace the connection even if caller work is still in flight on it. /// Why; for the trace. /// Bounds the wait for the write lock (together with the mutex's own timeout, whichever /// ends first); the drain re-waits until this deadline, so it is the deadline that decides how long overall. - /// null if the connection could not be retired yet - caller work in flight, the lock not - /// obtained inside the deadline, or the current connection not classifiable yet - so the drain should try - /// again; otherwise whether this handoff retired a connection. + /// null if the connection could not be retired yet - caller work in flight, or the lock not + /// obtained inside the deadline - so the drain should try again; otherwise whether this handoff retired a + /// connection. internal async ValueTask TryRetireTargetAsync(bool force, string reason, CancellationToken lockDeadline) { if (!TryGetHandoff(out var handoff)) return false; @@ -594,8 +522,7 @@ private static bool ShouldHoldBatch(List messages) // Lock-free first: the drain polls this every 20ms, and only a pass that is going to retire needs the write // lock (held across the check and the marking). The in-flight question needs only the connection's own - // lock; and with the target gone, finding the bridge disconnected, a replacement current, or a connection - // not yet classifiable settles the pass without a retire, so without the lock. The locked pass below + // lock; and with the target gone there is nothing to retire, so nothing to lock for. The locked pass below // repeats the judgement on what is current then, which is the one that counts. var current = physical; if (handoff.Target is { } drainTarget && ReferenceEquals(current, drainTarget)) @@ -604,16 +531,7 @@ private static bool ShouldHoldBatch(List messages) } else { - handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); - if (current is null) return false; - switch (Classify(handoff, current)) - { - case Replacement.Yes: - if (current.EverFullyEstablished) OnReplacementEstablished(handoff); - return false; - case Replacement.Unknown: - return null; - } + return TargetGone(handoff, current); } PhysicalConnection toRetire; @@ -637,43 +555,8 @@ private static bool ShouldHoldBatch(List messages) { if (handoff.Phase is HandoffPhase.Flushing or HandoffPhase.Done) return handoff.Retired; current = physical; - if (handoff.Target is { } target && ReferenceEquals(current, target)) - { - toRetire = target; - } - else - { - // The connection the hold began on is gone - it went away by itself (the server closed it, or it - // faulted), or there was none when the hold began because the bridge was already disconnected. - // The phase says so first, so that whatever is current is judged as of a target that is gone. - handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); - if (current is null) - { - // reconnecting: the reconnect is judged when it establishes, and whatever the hold diverted - // stays protected for it until then - return false; - } - switch (Classify(handoff, current)) - { - case Replacement.Yes: - // the reconnect is the replacement (a pinned successor, or demonstrably another node): - // retiring it would only force a second connect cycle. If it is already up, the hold is - // over now; if it is still connecting, its establishment will see to that. - if (current.EverFullyEstablished) OnReplacementEstablished(handoff); - return false; - case Replacement.Unknown: - // not classifiable yet - a socket still connecting, say (rule b): never retired on a guess, - // never branded a target; the next pass looks again, and the drain's deadline bounds this - return null; - default: - // back on the node being retired: what is current now is what the handoff replaces - - // unless its establishment or a heartbeat adopted it first, in which case that retire is - // under way and the next pass sees its result - if (!handoff.TryAdopt(current)) return null; - toRetire = current; - break; - } - } + if (handoff.Target is not { } target || !ReferenceEquals(current, target)) return TargetGone(handoff, current); + toRetire = target; if (toRetire.RetiredForHandoff) return handoff.Retired; // already retired by another path; its teardown follows that path's lock release if (!force && toRetire.HasCallerMessagesAwaitingResponse()) return null; @@ -702,6 +585,21 @@ private static bool ShouldHoldBatch(List messages) return true; } + /// + /// The connection the hold began on is gone - it went away by itself (the server closed it, or it faulted), or + /// there was none when the hold began because the bridge was already disconnected. The phase says so first, + /// so that whatever is current is judged as of a target that is gone: an established connection is the + /// replacement and the hold is over now; one still connecting is judged when it establishes; none means the + /// reconnect is coming, and whatever the hold diverted stays protected for it. Nothing to retire, so the + /// drain loop ends. + /// + private bool TargetGone(Handoff handoff, PhysicalConnection? current) + { + handoff.TryAdvance(HandoffPhase.Holding, HandoffPhase.AwaitingReplacement); + if (current is { EverFullyEstablished: true } && Classify(handoff, current) == Replacement.Yes) OnReplacementEstablished(handoff); + return false; + } + /// /// Tears down a connection a handoff has marked retired. Report before tearing down, and in this /// order for a reason: RecordConnectionFailed only raises the public event while the pipe is still live @@ -1193,22 +1091,13 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) connection.ReportAttemptOutcome(isSuccess: true, ConnectionFailureType.None, null); connection.MarkFullyEstablished(); connection.SetIdle(); - // Before the state changes to established, not after: a doomed connection becoming current while the - // held work is flushing must put the hold back in force before the processor can see a connected bridge - // with no hold and write to it. The judgement proper (adopt, retire) follows below, once established. - if (TryGetHandoff(out var early) && early.Phase == HandoffPhase.Flushing && physical == connection - && Classify(early, connection) == Replacement.Doomed) - { - early.TryAdvance(HandoffPhase.Flushing, HandoffPhase.AwaitingReplacement); - } if (physical == connection && !isDisposed && ChangeState(State.ConnectedEstablishing, State.ConnectedEstablished)) { reportNextFailure = reconfigureNextFailure = true; LastException = null; - // only a bona fide replacement establishing ends the hold for this bridge - not the very connection - // the hold is draining completing a handshake it was in the middle of when the hold began, and not a - // reconnect that landed back on the node being retired, which is retired in its turn (rule c: the - // connection's own facts decide) + // a replacement establishing ends the hold for this bridge - not the very connection the hold is + // draining completing a handshake it was in the middle of when the hold began (rule c: the connection + // says what it is) if (TryGetHandoff(out var handoff)) Judge(handoff, connection); Interlocked.Exchange(ref failConnectCount, 0); ServerEndPoint.OnFullyEstablished(connection, source); diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index 4e9a274d0..081e13508 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -316,9 +316,6 @@ internal async Task BeginConnectAsync(ILogger? log) { Trace($"handoff: connecting to {Format.ToString(handoffTarget)} in place of {Format.ToString(endpoint)}"); connectTo = handoffTarget; - // only where the socket below really dials it: a tunnel connects to the configured endpoint on its - // own terms, and recording a dial that never happened would be false evidence for the handoff - if (tunnel is null) _handoffTargetDialed = handoffTarget; } if (tunnel is not null) { @@ -995,7 +992,6 @@ internal void GetStormLog(StringBuilder sb) // volatile: each is written on one thread and read on others as part of the handoff's identity test, in the // very races that test exists for; a plain field could be read stale exactly then private volatile bool _wasHandoffTarget, _everFullyEstablished; - private volatile System.Net.EndPoint? _handoffTargetDialed; /// /// Whether this connection was ever the one a maintenance handoff held work back from. A durable fact about the @@ -1007,19 +1003,9 @@ internal void GetStormLog(StringBuilder sb) internal void MarkHandoffTarget() => _wasHandoffTarget = true; - /// - /// The successor a MOVING named (see ) that this connection was - /// dialled to, in place of the endpoint as resolved; null if it was dialled to the endpoint. Recorded as the - /// socket is pointed there, so a handoff judging the connection later does not have to infer it from whether - /// the pin is still in force - and recorded as the address rather than a flag, so that the judgement can be - /// scoped to the handoff whose successor it is: a connection dialled through an earlier handoff's pin is not - /// thereby the replacement for a later one. - /// - internal System.Net.EndPoint? HandoffTargetDialed => _handoffTargetDialed; - /// /// The socket's remote address and port, or null if the socket is not connected yet, is already gone, or - /// there is no socket. Cached once read: a connected socket's remote endpoint never changes. + /// there is no socket. Read when a MOVING arrives, to tell the handoff's DNS poll where we are now. /// internal System.Net.IPEndPoint? TryGetRemoteEndPoint() { diff --git a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs index 63a59926b..e99a4b898 100644 --- a/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs +++ b/src/StackExchange.Redis/ServerEndPoint.Maintenance.cs @@ -315,7 +315,6 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? return; } - Volatile.Write(ref _relaxedType, (int)type); var tail = Multiplexer.RawConfig.MaintenancePostEventRelaxedDuration; // While a handoff is live - from the hold beginning until the held work has reached the replacement - the @@ -323,16 +322,30 @@ internal void OnMaintenanceWindowClosed(MaintenanceNotificationType type, long? // operation arriving meanwhile (a MIGRATED for a shard move) must not pull the deadline back under it and // have the sweep fail the very commands the handoff is carrying across. So during a handoff a close can // lengthen the window but not shorten it; the replace semantics above apply the rest of the time. + // The type reported on faults follows the window it describes: it changes only when this completion is what + // the window now stands for - not when a handoff's window is preserved past it, or a timeout the handoff + // covered would be attributed to an operation that had finished. var handoffInFlight = HandoffLive; if (tail <= TimeSpan.Zero) { - if (!handoffInFlight) _relaxedDeadline.SetIn(TimeSpan.Zero); // ended now, and recorded as such for fault attribution + if (!handoffInFlight) + { + _relaxedDeadline.SetIn(TimeSpan.Zero); // ended now, and recorded as such for fault attribution + Volatile.Write(ref _relaxedType, (int)type); + } Multiplexer.Trace($"{type}: relaxation ended", ToString()); return; } - if (handoffInFlight) _relaxedDeadline.TryExtendBy(tail, out _); - else _relaxedDeadline.SetIn(tail); + if (!handoffInFlight) + { + _relaxedDeadline.SetIn(tail); + Volatile.Write(ref _relaxedType, (int)type); + } + else if (_relaxedDeadline.TryExtendBy(tail, out _)) + { + Volatile.Write(ref _relaxedType, (int)type); + } Multiplexer.Trace($"{type}: relaxation continues for {tail.TotalSeconds}s (post-event)", ToString()); } @@ -390,7 +403,7 @@ private void SetHandoffTarget(EndPoint target, TimeSpan window, int generation) { lock (_maintenanceSync) { - if (_handoffTarget is not null && generation < _handoffTargetGeneration) return; + if (generation < _handoffTargetGeneration) return; // whether or not that later pin has since expired: it knew more _handoffTargetExpiry.SetIn(window < TimeSpan.FromSeconds(1) ? TimeSpan.FromSeconds(1) : window); _handoffTarget = target; _handoffTargetGeneration = generation; @@ -406,8 +419,9 @@ private void ClearHandoffTarget(int generation) { lock (_maintenanceSync) { - if (_handoffTarget is not null && generation < _handoffTargetGeneration) return; + if (generation < _handoffTargetGeneration) return; _handoffTarget = null; + _handoffTargetGeneration = generation; } } @@ -464,7 +478,8 @@ internal void OnMovingAnnounced(TimeSpan? window, EndPoint? successor, PhysicalC if (Interlocked.CompareExchange(ref _handoffInFlight, 1, 0) != 0) { if (successor is not null) SetHandoffTarget(successor, budget, generation); - Multiplexer.Trace(successor is null ? "MOVING: a handoff is already in flight" : "MOVING: a handoff is already in flight; successor noted", ToString()); + else ClearHandoffTarget(generation); // as on the direct path: no successor named means any pinned node may itself be retiring + Multiplexer.Trace(successor is null ? "MOVING: a handoff is already in flight; pin dropped" : "MOVING: a handoff is already in flight; successor noted", ToString()); return; } var current = connection.TryGetRemoteEndPoint(); // null if the socket is gone or not connected: nothing to compare against @@ -511,7 +526,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // DNS has moved to a new address, so a reconnect reporting the old one is back on the retiring // node; a pin left by an earlier handoff is stale now and must not dial it ClearHandoffTarget(generation); - await DrainThenRecycleAsync(remaining, decision.Reason, successor: null, replacementAddressSeen: true, currentEndPoint).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); replaced = true; break; case HandoffAction.RecycleAtHalfWindow: @@ -521,7 +536,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint if (half > TimeSpan.Zero) await Task.Delay(half).ForAwait(); // nothing to re-resolve, or nothing known to compare against: no second address has been seen ClearHandoffTarget(generation); - await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason, successor: null, replacementAddressSeen: false, currentEndPoint).ForAwait(); + await DrainThenRecycleAsync(Remaining(window, watch), decision.Reason).ForAwait(); replaced = true; break; case HandoffAction.MoveTo when decision.Target is { } target: @@ -530,11 +545,7 @@ private async Task HandoffAsync(TimeSpan window, EndPoint? successor, IPEndPoint // at +6.2s, landed back on the node being retired because DNS had not moved yet, and were // closed at +21.6s anyway - exactly the outcome the handoff exists to avoid. SetHandoffTarget(target, remaining, generation); - // A successor at the very address being retired - a proxy restarted in place - means the address - // identifies nothing for this handoff: every connection, doomed or replacement, reports it. The - // pin-dialled test then governs, and after that only time. - var sameAddress = target is IPEndPoint successorAddress && currentEndPoint is not null && successorAddress.Equals(currentEndPoint); - await DrainThenRecycleAsync(remaining, decision.Reason, target, replacementAddressSeen: !sameAddress, currentEndPoint).ForAwait(); + await DrainThenRecycleAsync(remaining, decision.Reason).ForAwait(); replaced = true; break; default: @@ -656,16 +667,7 @@ private TimeSpan ClampToRelaxationCeiling(TimeSpan span, TimeSpan floor) /// /// What is left of the announced window. /// Why; for the trace. - /// The successor the server named and reconnection is pinned to, if any: a connection - /// dialled through it is the replacement. - /// Whether the handoff has seen a second address - the server named a - /// successor at a different address, or DNS moved to a new address - so that a reconnect reporting the retiring - /// address is known to be back on the retiring node, and is retired in its turn; see - /// . Where no second address was ever involved (including a - /// successor at the same address), the address says nothing and only time tells. - /// The address and port of the node being retired, as seen from the connection that - /// was told, so that a replacement which landed elsewhere is recognised as such. - private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, EndPoint? successor, bool replacementAddressSeen, IPEndPoint? retiringEndPoint) + private async Task DrainThenRecycleAsync(TimeSpan budget, string reason) { var watch = ValueStopwatch.StartNew(); // With less than the floor left of the window there is no drain worth having: the server is about to close @@ -694,15 +696,8 @@ private async Task DrainThenRecycleAsync(TimeSpan budget, string reason, EndPoin var interactiveBridge = interactive; var subscriptionBridge = subscription; - // Whether a socket's remote address says anything about *which node* it reached: only when this handoff has - // itself seen a second address, and there is no tunnel in the way. Behind a Tunnel every connection reports - // the tunnel's address; and where the only address ever involved is the one in front of the cluster (a load - // balancer or VIP) comparing it with the retiring one would make the genuine replacement look like a return - // to the retiring node forever. Then only time can tell: once the target is gone, what is current is the - // replacement. - var addressIdentifiesNode = replacementAddressSeen && Multiplexer.RawConfig.Tunnel is null; - interactiveBridge?.BeginHold(cover, retiringEndPoint, successor, addressIdentifiesNode); - subscriptionBridge?.BeginHold(cover, retiringEndPoint, successor, addressIdentifiesNode); + interactiveBridge?.BeginHold(cover); + subscriptionBridge?.BeginHold(cover); // One loop per bridge, each with its own lock deadline, run side by side: a writer wedged on one bridge's // lock must not spend the other's time, and the other's idle connection must still be retired before the diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 3b8ad6246..8382b92c5 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -873,6 +873,112 @@ public Task MovingUnderSteadyLoadWithShortTimeoutsAndRelaxationOffKeepsHeldWork( latencyOnlyFor: RedisCommand.PING, workers: 2); // two in flight at most: serially served, their ordinary wait stays inside the timeout + /// When, in the first handoff's life, the second MOVING arrives. + public enum SecondMovingAt + { + /// At once: the first handoff is still in its jitter, deciding. + Decision, + + /// Inside the first drain, kept open by a slow command in flight. + Drain, + + /// After the first recycle, while the held work reaches the replacement. + Flush, + } + + [Theory] + [InlineData(true, SecondMovingAt.Decision)] + [InlineData(false, SecondMovingAt.Decision)] + [InlineData(true, SecondMovingAt.Drain)] + [InlineData(false, SecondMovingAt.Drain)] + [InlineData(true, SecondMovingAt.Flush)] + [InlineData(false, SecondMovingAt.Flush)] + public async Task MovingTwiceInOneWindowLosesNothing(bool secondNamesSuccessor, SecondMovingAt at) + { + // The contract for back-to-back MOVINGs inside one window is best effort (see docs/ServerMaintenanceEvent.md): + // nothing is lost or abandoned contrary to the backlog policy, and traffic resumes within the handoff's own + // bound. Which successor the pin ends up naming, and whether the second notification starts a drain of its own + // or only redirects the first's reconnect, are not asserted: they are the mechanism, not the promise. + var logs = new CapturingLoggerFactory(); + var (server, conn, events) = await ConnectAsync(log, logs); + using (server) + await using (conn) + { + server.SetLatency(TimeSpan.FromMilliseconds(40)); + var db = conn.GetDatabase(); + using var stop = new CancellationTokenSource(); + long completed = 0; + var failures = new List(); + var workerTasks = Enumerable.Range(0, 4).Select(_ => Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + try + { + await db.PingAsync(); + Interlocked.Increment(ref completed); + } + catch (Exception ex) + { + lock (failures) failures.Add(ex); + } + } + })).ToArray(); + try + { + Assert.True(await Poll.UntilAsync(() => Volatile.Read(ref completed) > 20, timeoutMilliseconds: 30_000), "the load should be flowing before the handoff"); + var endpoint = ((IInternalConnectionMultiplexer)conn).GetServerEndPoint(server.DefaultEndPoint); + var first = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); + var second = secondNamesSuccessor ? new IPEndPoint(IPAddress.Parse("127.0.0.10"), 6380) : null; + + Task? slow = null; + if (at == SecondMovingAt.Drain) + { + // one slow command in flight keeps the first drain open (3s, inside the 5s cap) for the second MOVING to land in + server.SetLatency(TimeSpan.FromSeconds(3), RedisCommand.GET); + slow = db.StringGetAsync("slow"); + Assert.True(await Poll.UntilAsync(() => conn.GetCounters().Interactive.SentItemsAwaitingResponse > 0), "the slow command should be in flight before MOVING arrives"); + } + + var watch = System.Diagnostics.Stopwatch.StartNew(); + server.SendMoving(null, timeSeconds: 10, newEndpoint: first, sequenceId: 0); + Assert.Equal(MaintenanceNotificationType.Moving, (await events.NextAsync()).NotificationType); + + switch (at) + { + case SecondMovingAt.Drain: + await Task.Delay(1500); // past the jitter (at most a tenth of the window), inside the drain the slow command holds open + break; + case SecondMovingAt.Flush: + Assert.True(await Poll.UntilAsync(() => endpoint.HandoffRecycles > 0, timeoutMilliseconds: 8_000), "the first handoff should have replaced the connection"); + break; + } + server.SendMoving(null, timeSeconds: 10, newEndpoint: second, sequenceId: 1); + Assert.Equal(MaintenanceNotificationType.Moving, (await events.NextAsync()).NotificationType); + log.WriteLine($"second MOVING ({(second is null ? "no successor" : "successor named")}) sent at +{watch.ElapsedMilliseconds}ms, during {at}; pin now {endpoint.HandoffTarget?.ToString() ?? "(none)"}"); + + var before = Volatile.Read(ref completed); + Assert.True( + await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutMilliseconds: 20_000), + "traffic should resume after the second MOVING"); + log.WriteLine($"traffic flowing again at +{watch.ElapsedMilliseconds}ms; recycles={endpoint.HandoffRecycles}; pin {endpoint.HandoffTarget?.ToString() ?? "(none)"}"); + + if (slow is not null) await slow; // drained, not failed: it fitted inside the cap + stop.Cancel(); + await Task.WhenAll(workerTasks); + lock (failures) + { + log.WriteLine($"completed: {Volatile.Read(ref completed)}; failed: {failures.Count}"); + Assert.Empty(failures.Select(f => f.Message)); + } + } + finally + { + stop.Cancel(); + } + } + } + private async Task SteadyLoadHandoffAsync(Action? configure, int serverLatencyMilliseconds = 40, RedisCommand? latencyOnlyFor = null, int workers = 4) { // The drain used to wait for "no caller work", which under steady traffic never comes: measured live at From 8cddb487f86a1d03ea01062e925acad70061f26e Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 6 Nov 2026 09:26:36 +0000 Subject: [PATCH 26/27] Handoff: the post-enqueue re-stamp no longer depends on the first read finding no handoff A message stamped under a still-live handoff while a later hold begins is re-stamped with the later one's deadline (the stamp only extends; a same-handoff double stamp is a no-op). The last item from the exit assessment. --- src/StackExchange.Redis/PhysicalBridge.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 6b68aec1a..5c7b596f1 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -1576,9 +1576,11 @@ private void BacklogEnqueue(Message message) // The total is counted after the enqueue, deliberately: an interlocked increment is a full fence, and the // re-read below needs one after the enqueue's store. Enqueue, fence, re-read: a hold that began between // the read above and the enqueue has walked the queue without this message, or is about to walk it with - // it; either way it ends up stamped. A bridge that has never seen a handoff pays one null read for this. + // it; either way it ends up stamped - with the later handoff's deadline if the two differ, since the stamp + // only ever extends and a same-handoff double stamp is a no-op. A bridge that has never seen a handoff + // pays one null read for this. Interlocked.Increment(ref _backlogTotalEnqueued); - if (holdable && !live && _handoff is not null && TryGetHandoff(out var late)) message.MarkHeldForHandoff(late.Deadline.Ticks); + if (holdable && _handoff is not null && TryGetHandoff(out var late)) message.MarkHeldForHandoff(late.Deadline.Ticks); if (wasEmpty) { From 7d17cbeddde9f097d7240afebd4db79089a08e10 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 6 Nov 2026 10:54:05 +0000 Subject: [PATCH 27/27] Tests: the two-MOVINGs matrix keeps its workers throttled in the drain cases A per-command latency lifts the throttle from every other command, so the four PING workers hot-looped for ten seconds in the two Drain cases; on a small CI runner that starved timing-sensitive neighbours (three different tests failing once each across two runs). Every reply is slowed instead, and the throttle is restored once the second MOVING is in. --- .../MaintenanceNotificationTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs index 8382b92c5..9631837da 100644 --- a/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs +++ b/tests/StackExchange.Redis.Tests/MaintenanceNotificationTests.cs @@ -931,13 +931,14 @@ public async Task MovingTwiceInOneWindowLosesNothing(bool secondNamesSuccessor, var first = new IPEndPoint(IPAddress.Parse("127.0.0.9"), 6380); var second = secondNamesSuccessor ? new IPEndPoint(IPAddress.Parse("127.0.0.10"), 6380) : null; - Task? slow = null; if (at == SecondMovingAt.Drain) { - // one slow command in flight keeps the first drain open (3s, inside the 5s cap) for the second MOVING to land in - server.SetLatency(TimeSpan.FromSeconds(3), RedisCommand.GET); - slow = db.StringGetAsync("slow"); - Assert.True(await Poll.UntilAsync(() => conn.GetCounters().Interactive.SentItemsAwaitingResponse > 0), "the slow command should be in flight before MOVING arrives"); + // slow replies keep the first drain open (3s, inside the 5s cap) for the second MOVING to land in. Every + // command, not one: a per-command latency lifts the throttle from the workers' PINGs, and four + // unthrottled workers for ten seconds starve the timing-sensitive tests running alongside on a small + // CI runner (observed: three different neighbours failing once each across two runs) + server.SetLatency(TimeSpan.FromSeconds(3)); + Assert.True(await Poll.UntilAsync(() => conn.GetCounters().Interactive.SentItemsAwaitingResponse > 0), "slow commands should be in flight before MOVING arrives"); } var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -955,6 +956,7 @@ public async Task MovingTwiceInOneWindowLosesNothing(bool secondNamesSuccessor, } server.SendMoving(null, timeSeconds: 10, newEndpoint: second, sequenceId: 1); Assert.Equal(MaintenanceNotificationType.Moving, (await events.NextAsync()).NotificationType); + if (at == SecondMovingAt.Drain) server.SetLatency(TimeSpan.FromMilliseconds(40)); // the replies already being held keep their 3s; what follows flows log.WriteLine($"second MOVING ({(second is null ? "no successor" : "successor named")}) sent at +{watch.ElapsedMilliseconds}ms, during {at}; pin now {endpoint.HandoffTarget?.ToString() ?? "(none)"}"); var before = Volatile.Read(ref completed); @@ -963,7 +965,6 @@ await Poll.UntilAsync(() => Volatile.Read(ref completed) > before + 20, timeoutM "traffic should resume after the second MOVING"); log.WriteLine($"traffic flowing again at +{watch.ElapsedMilliseconds}ms; recycles={endpoint.HandoffRecycles}; pin {endpoint.HandoffTarget?.ToString() ?? "(none)"}"); - if (slow is not null) await slow; // drained, not failed: it fitted inside the cap stop.Cancel(); await Task.WhenAll(workerTasks); lock (failures)