From c58816a793af65f7f7eb3ba65538e3edc65584ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Litfin?= Date: Thu, 12 Nov 2026 10:16:42 +0100 Subject: [PATCH] Serialize remote event registration and requeue operations --- repros/RemoteEventRequeue/Program.cs | 169 ++++++++++++++++++ repros/RemoteEventRequeue/README.md | 52 ++++++ .../RemoteEventRequeueRepro.csproj | 9 + .../RemoteEventRequeue/operation-probe.patch | 49 +++++ .../FbRemoteEventTests.cs | 41 +++++ .../Client/Managed/Version10/GdsDatabase.cs | 75 ++++++++ 6 files changed, 395 insertions(+) create mode 100644 repros/RemoteEventRequeue/Program.cs create mode 100644 repros/RemoteEventRequeue/README.md create mode 100644 repros/RemoteEventRequeue/RemoteEventRequeueRepro.csproj create mode 100644 repros/RemoteEventRequeue/operation-probe.patch diff --git a/repros/RemoteEventRequeue/Program.cs b/repros/RemoteEventRequeue/Program.cs new file mode 100644 index 000000000..c32b02255 --- /dev/null +++ b/repros/RemoteEventRequeue/Program.cs @@ -0,0 +1,169 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using FirebirdSql.Data.FirebirdClient; + +namespace RemoteEventRequeueRepro +{ + internal static class Program + { + private const string EventName = "NETPROVIDER_EVENT_REPRO"; + + private static int Main(string[] args) + { + string rawConnectionString = Environment.GetEnvironmentVariable("FB_EVENT_REPRO_CONNECTION"); + if (string.IsNullOrWhiteSpace(rawConnectionString)) + { + Console.Error.WriteLine("FB_EVENT_REPRO_CONNECTION is missing. Use only an isolated test database."); + return 2; + } + + int iterations = ReadPositiveInteger(args, 0, 1000); + int holdMilliseconds = ReadPositiveInteger(args, 1, 10); + var builder = new FbConnectionStringBuilder(rawConnectionString) { Pooling = false }; + Console.WriteLine("Target=" + builder.DataSource + ":" + builder.Port + + ", Database=" + builder.Database + ", ServerType=" + builder.ServerType); + string connectionString = builder.ToString(); + using (var connectionProbe = new FbConnection(connectionString)) + { + Console.WriteLine("Connection probe: Open"); + connectionProbe.Open(); + Console.WriteLine("Connection probe: connected"); + } + int published = 0; + int received = 0; + int queueErrors = 0; + int cleanupErrors = 0; + int eventErrors = 0; + int completed = 0; + + using (var stop = new CancellationTokenSource()) + using (var publisherReady = new ManualResetEventSlim(false)) + { + Task publisher = Task.Factory.StartNew( + () => PublishEvents(connectionString, publisherReady, stop.Token, + () => Interlocked.Increment(ref published)), + CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + Console.WriteLine("Waiting for publisher connection."); + if (!publisherReady.Wait(TimeSpan.FromSeconds(20))) + { + Console.Error.WriteLine("Publisher connection not ready after 20 seconds."); + return 2; + } + if (publisher.IsFaulted) + { + Console.Error.WriteLine("Publisher failed to start: " + publisher.Exception.GetBaseException()); + return 2; + } + + for (int i = 0; i < iterations; i++) + { + FbRemoteEvent events = null; + bool queued = false; + try + { + events = new FbRemoteEvent(connectionString); + events.RemoteEventCounts += (sender, e) => Interlocked.Increment(ref received); + events.RemoteEventError += (sender, e) => + { + Interlocked.Increment(ref eventErrors); + Console.Error.WriteLine("Event error: " + e.Error); + }; + Console.WriteLine("Iteration " + (i + 1) + ": Open"); + events.Open(); + Console.WriteLine("Iteration " + (i + 1) + ": QueueEvents"); + events.QueueEvents(new[] { EventName }); + Console.WriteLine("Iteration " + (i + 1) + ": registered"); + queued = true; + Thread.Sleep(holdMilliseconds); + } + catch (Exception ex) + { + queueErrors++; + Console.Error.WriteLine("Iteration " + (i + 1) + ", Open/QueueEvents: " + ex); + } + finally + { + if (events != null) + { + if (queued) + { + try { events.CancelEvents(); } + catch (Exception ex) + { + cleanupErrors++; + Console.Error.WriteLine("Iteration " + (i + 1) + ", CancelEvents: " + ex); + } + } + try { events.Dispose(); } + catch (Exception ex) + { + cleanupErrors++; + Console.Error.WriteLine("Iteration " + (i + 1) + ", Dispose: " + ex); + } + } + } + completed++; + + if (publisher.IsFaulted) + break; + } + + stop.Cancel(); + try { publisher.GetAwaiter().GetResult(); } + catch (Exception ex) + { + Console.Error.WriteLine("Publisher error: " + ex); + return 2; + } + } + + Console.WriteLine("Provider=" + typeof(FbRemoteEvent).Assembly.GetName().Version + + ", Iterations=" + completed + ", sent=" + published + + ", received=" + received + ", Queue errors=" + queueErrors + + ", Event errors=" + eventErrors + ", Cleanup errors=" + cleanupErrors); + return queueErrors == 0 && eventErrors == 0 && cleanupErrors == 0 ? 0 : 1; + } + + private static void PublishEvents(string connectionString, ManualResetEventSlim ready, + CancellationToken stop, Action published) + { + try + { + Console.WriteLine("Publisher: create connection"); + using (var connection = new FbConnection(connectionString)) + { + Console.WriteLine("Publisher: Open"); + connection.Open(); + Console.WriteLine("Publisher: connected"); + ready.Set(); + while (!stop.IsCancellationRequested) + { + using (var transaction = connection.BeginTransaction()) + using (var command = new FbCommand()) + { + command.Connection = connection; + command.Transaction = transaction; + command.CommandText = "EXECUTE BLOCK AS BEGIN POST_EVENT '" + EventName + "'; END"; + command.ExecuteNonQuery(); + transaction.Commit(); + } + published(); + Thread.Sleep(1); + } + } + } + finally + { + ready.Set(); + } + } + + private static int ReadPositiveInteger(string[] args, int index, int defaultValue) + { + int result; + return args.Length > index && int.TryParse(args[index], out result) && result > 0 + ? result : defaultValue; + } + } +} diff --git a/repros/RemoteEventRequeue/README.md b/repros/RemoteEventRequeue/README.md new file mode 100644 index 000000000..5f647ee77 --- /dev/null +++ b/repros/RemoteEventRequeue/README.md @@ -0,0 +1,52 @@ +# Remote event requeue overlapping the initial registration + +This reproducer accompanies the fix for an internally initiated overlap in the managed provider. It is related to [issue #963](https://github.com/FirebirdSQL/NETProvider/issues/963), which reports `ArgumentOutOfRangeException: operation=`. It does **not** establish that every report in that issue has this cause. In particular, this reproducer uses a separate connection for publishing events; it does not issue parallel application commands on one `FbConnection`. + +## What overlaps + +`FbRemoteEvent.QueueEvents` sends `op_que_events` on the event database's main connection and reads its response synchronously. After the first event arrives on the auxiliary event socket, `GdsEventManager.StartWaitingForEvents` calls `GdsDatabase.QueueEventsAsync` to register the next notification on **that same main protocol stream**. If the synchronous operation has not finished consuming its response, the asynchronous read can consume bytes belonging to the synchronous response. `GdsConnection.ProcessOperationAsync` then receives an invalid operation code and throws `ArgumentOutOfRangeException: operation=0` (the number depends on which bytes were consumed). + +The two operations originate inside the provider even when the application uses `FbRemoteEvent` correctly. `CancelEvents` and `Detach` use the same stream and must not enter while an automatic requeue is in flight. The fix serializes those complete request/response exchanges per `GdsDatabase`, across sync and async variants. It does not change the general rule that application code must not perform concurrent commands on a single `FbConnection`. + +## Controlled reproduction on the unfixed revision + +The race is timing dependent. `operation-probe.patch` is a **test-only scheduling probe**, not part of the production fix. Apply it to a separate checkout of the unfixed revision. It pauses the synchronous response reader after obtaining the first opcode and lets the automatic asynchronous requeue read the following bytes. The probe is enabled only with `FB_EVENT_OPERATION_PROBE=1`. Never include this patch in a provider build intended for users. + +Requirements: .NET 10 SDK, Firebird server with remote events enabled, and an isolated disposable database. We used Firebird 3.0.12 on `127.0.0.1:3051`. Set `FB_EVENT_REPRO_CONNECTION` to a connection string for that database. The runner disables pooling and uses separate connections for event registration and publishing. It posts only a named event and creates no tables. + +```powershell +git clone https://github.com/FirebirdSQL/NETProvider.git NETProvider-unfixed +cd NETProvider-unfixed +git checkout 4385c0fdd6641877cf10bf9d3c81305428a743ef +git apply repros\RemoteEventRequeue\operation-probe.patch +$env:FB_EVENT_OPERATION_PROBE = '1' +$env:FB_EVENT_REPRO_CONNECTION = '' +dotnet build repros\RemoteEventRequeue\RemoteEventRequeueRepro.csproj -p:TargetFrameworks=net10.0 -p:GeneratePackageOnBuild=false +dotnet repros\RemoteEventRequeue\bin\Debug\net10.0\RemoteEventRequeueRepro.dll 1 10 +``` + +Copy this repro directory into the `repros` folder of the unfixed checkout first, so its project reference points to that checkout. The target-framework override is needed if the local SDK cannot build the provider's other target frameworks. + +In our run against revision `4385c0fd`, the first event was delivered and `RemoteEventError` reported: + +```text +System.ArgumentOutOfRangeException: operation=0 + at GdsConnection.ProcessOperationAsync(...) + at GdsDatabase.ReadSingleResponseAsync(...) + at GdsDatabase.QueueEventsAsync(...) + at GdsEventManager.StartWaitingForEvents(...) +``` + +The runner exited with an event error. This controlled probe establishes that overlapping response reads can produce the reported exception signature. It does not prove that an uninstrumented production occurrence had this exact timing. + +## Validation of the fix + +Without the probe, run the same program from this branch for a longer event/registration/cancellation stress run: + +```powershell +$env:FB_EVENT_REPRO_CONNECTION = '' +dotnet build repros\RemoteEventRequeue\RemoteEventRequeueRepro.csproj -p:TargetFrameworks=net10.0 -p:GeneratePackageOnBuild=false +dotnet repros\RemoteEventRequeue\bin\Debug\net10.0\RemoteEventRequeueRepro.dll 1000 10 +``` + +On the same Firebird 3.0.12 test database, the fixed provider completed 1,000 iterations with 2,029 events published, 957 received, and zero queue, event, or cleanup errors. The provider and test project compiled for .NET 10. The automated `FbRemoteEventTests.AutoRequeueDuringInitialRegistrationAndCancellationTest` exercises the same pattern in the normal integration suite. The local machine had no .NET 11 SDK and did not run the full suite or test against Firebird 2.5. diff --git a/repros/RemoteEventRequeue/RemoteEventRequeueRepro.csproj b/repros/RemoteEventRequeue/RemoteEventRequeueRepro.csproj new file mode 100644 index 000000000..9c966cb03 --- /dev/null +++ b/repros/RemoteEventRequeue/RemoteEventRequeueRepro.csproj @@ -0,0 +1,9 @@ + + + Exe + net10.0 + + + + + \ No newline at end of file diff --git a/repros/RemoteEventRequeue/operation-probe.patch b/repros/RemoteEventRequeue/operation-probe.patch new file mode 100644 index 000000000..cdb32a111 --- /dev/null +++ b/repros/RemoteEventRequeue/operation-probe.patch @@ -0,0 +1,49 @@ +diff --git a/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs b/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs +index 37f134aa..673740d3 100644 +--- a/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs ++++ b/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs +@@ -39,6 +39,7 @@ internal class GdsDatabase : DatabaseBase + protected GdsConnection _connection; + protected GdsEventManager _eventManager; + protected int _handle; ++ private readonly ManualResetEventSlim _asyncReadFinishedForProbe = new(false); + + #endregion + +@@ -564,6 +565,15 @@ internal class GdsDatabase : DatabaseBase + Xdr.Write(remoteEvent.LocalId); + + Xdr.Flush(); ++ if (Environment.GetEnvironmentVariable("FB_EVENT_OPERATION_PROBE") == "1") ++ { ++ var firstOpcode = ReadOperation(); ++ if (!_asyncReadFinishedForProbe.Wait(TimeSpan.FromSeconds(10))) ++ throw new TimeoutException("The asynchronous event requeue did not read the first response."); ++ var firstResponse = (GenericResponse)ReadResponse(firstOpcode); ++ remoteEvent.RemoteId = firstResponse.ObjectHandle; ++ return; ++ } + + var response = (GenericResponse)ReadResponse(); + +@@ -599,6 +609,20 @@ internal class GdsDatabase : DatabaseBase + await Xdr.WriteAsync(remoteEvent.LocalId, cancellationToken).ConfigureAwait(false); + + await Xdr.FlushAsync(cancellationToken).ConfigureAwait(false); ++ if (Environment.GetEnvironmentVariable("FB_EVENT_OPERATION_PROBE") == "1") ++ { ++ try ++ { ++ var wrongOpcode = await ReadOperationAsync(cancellationToken).ConfigureAwait(false); ++ var wrongResponse = (GenericResponse)await ReadResponseAsync(wrongOpcode, cancellationToken).ConfigureAwait(false); ++ remoteEvent.RemoteId = wrongResponse.ObjectHandle; ++ return; ++ } ++ finally ++ { ++ _asyncReadFinishedForProbe.Set(); ++ } ++ } + + var response = (GenericResponse)await ReadResponseAsync(cancellationToken).ConfigureAwait(false); + diff --git a/src/FirebirdSql.Data.FirebirdClient.Tests/FbRemoteEventTests.cs b/src/FirebirdSql.Data.FirebirdClient.Tests/FbRemoteEventTests.cs index 6507c6544..4a1c5644c 100644 --- a/src/FirebirdSql.Data.FirebirdClient.Tests/FbRemoteEventTests.cs +++ b/src/FirebirdSql.Data.FirebirdClient.Tests/FbRemoteEventTests.cs @@ -16,6 +16,7 @@ //$Authors = Jiri Cincura (jiri@cincura.net) using System; +using System.Collections.Concurrent; using System.Threading; using System.Threading.Tasks; using FirebirdSql.Data.TestsBase; @@ -30,6 +31,46 @@ public FbRemoteEventTests(FbServerType serverType, bool compression, FbWireCrypt : base(serverType, compression, wireCrypt) { } + [Test] + public async Task AutoRequeueDuringInitialRegistrationAndCancellationTest() + { + var errors = new ConcurrentQueue(); + var received = 0; + await using var publisherConnection = new FbConnection(Connection.ConnectionString); + await publisherConnection.OpenAsync(); + using var stop = new CancellationTokenSource(); + var publisher = Task.Run(async () => + { + await using var command = publisherConnection.CreateCommand(); + command.CommandText = "execute block as begin post_event 'test'; end"; + while (!stop.IsCancellationRequested) + { + await command.ExecuteNonQueryAsync(); + await Task.Delay(1); + } + }); + try + { + for (var i = 0; i < 100; i++) + { + await using var @event = new FbRemoteEvent(Connection.ConnectionString); + @event.RemoteEventError += (_, e) => errors.Enqueue(e.Error); + @event.RemoteEventCounts += (_, _) => Interlocked.Increment(ref received); + @event.Open(); + @event.QueueEvents(new[] { "test" }); + await Task.Delay(10); + @event.CancelEvents(); + } + } + finally + { + stop.Cancel(); + await publisher; + } + Assert.Greater(received, 0); + Assert.AreEqual(0, errors.Count); + } + [Test] public async Task EventSimplyComesBackTest() { diff --git a/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs b/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs index 37f134aa1..bf68b395b 100644 --- a/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs +++ b/src/FirebirdSql.Data.FirebirdClient/Client/Managed/Version10/GdsDatabase.cs @@ -39,6 +39,9 @@ internal class GdsDatabase : DatabaseBase protected GdsConnection _connection; protected GdsEventManager _eventManager; protected int _handle; + // The event listener can requeue asynchronously while QueueEvents is still reading its response. + // Keep complete event request/response exchanges on this database's protocol stream together. + private readonly SemaphoreSlim _eventOperationGate = new(1, 1); #endregion @@ -181,6 +184,18 @@ public override ValueTask AttachWithTrustedAuthAsync(DatabaseParameterBufferBase } public override void Detach() + { + _eventOperationGate.Wait(); + try + { + DetachCore(); + } + finally + { + _eventOperationGate.Release(); + } + } + private void DetachCore() { if (TransactionCount > 0) { @@ -227,6 +242,18 @@ public override void Detach() } } public override async ValueTask DetachAsync(CancellationToken cancellationToken = default) + { + await _eventOperationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await DetachCoreAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + _eventOperationGate.Release(); + } + } + private async ValueTask DetachCoreAsync(CancellationToken cancellationToken) { if (TransactionCount > 0) { @@ -540,6 +567,18 @@ public override async ValueTask CloseEventManagerAsync(CancellationToken cancell } public override void QueueEvents(RemoteEvent remoteEvent) + { + _eventOperationGate.Wait(); + try + { + QueueEventsCore(remoteEvent); + } + finally + { + _eventOperationGate.Release(); + } + } + private void QueueEventsCore(RemoteEvent remoteEvent) { try { @@ -575,6 +614,18 @@ public override void QueueEvents(RemoteEvent remoteEvent) } } public override async ValueTask QueueEventsAsync(RemoteEvent remoteEvent, CancellationToken cancellationToken = default) + { + await _eventOperationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await QueueEventsCoreAsync(remoteEvent, cancellationToken).ConfigureAwait(false); + } + finally + { + _eventOperationGate.Release(); + } + } + private async ValueTask QueueEventsCoreAsync(RemoteEvent remoteEvent, CancellationToken cancellationToken) { try { @@ -611,6 +662,18 @@ public override async ValueTask QueueEventsAsync(RemoteEvent remoteEvent, Cancel } public override void CancelEvents(RemoteEvent events) + { + _eventOperationGate.Wait(); + try + { + CancelEventsCore(events); + } + finally + { + _eventOperationGate.Release(); + } + } + private void CancelEventsCore(RemoteEvent events) { try { @@ -628,6 +691,18 @@ public override void CancelEvents(RemoteEvent events) } } public override async ValueTask CancelEventsAsync(RemoteEvent events, CancellationToken cancellationToken = default) + { + await _eventOperationGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await CancelEventsCoreAsync(events, cancellationToken).ConfigureAwait(false); + } + finally + { + _eventOperationGate.Release(); + } + } + private async ValueTask CancelEventsCoreAsync(RemoteEvent events, CancellationToken cancellationToken) { try {