From 8bde8ba58e7d300967823395b03d5c9d66e116f7 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:07:01 -1000 Subject: [PATCH 1/6] Wait for the bus's pull before deleting its JetStream consumer in tests Deleting a consumer answers the pulls waiting on it with 409 Consumer Deleted, but nats-server drops a pull that's still queued for the consumer as it closes, without an answer. The client then gives up on that pull only after missing two idle heartbeats, 30 seconds by default, which is the tests' own timeout. The deletion tests deleted right after the host started, while the bus's first pull could be on its way, so they sometimes timed out (the v7.0.0 tag build). They now wait until the consumer has a pull waiting before deleting it. Co-Authored-By: Claude Opus 5.5 --- .../JetStream/TestNatsJetStreamReconnect.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs index ce5bc9f6..a38b928e 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs @@ -157,8 +157,10 @@ public async Task ShouldReportAnErrorToSubscribersWhenTheSharedCommandConsumerIs .HandleCommand((command, context, ct) => Task.CompletedTask)); var errors = RecordErrors(worker.Services.GetRequiredService()); await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); + var jetStream = new NatsJSContext(connection); + await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Commands", ServiceName); - await new NatsJSContext(connection).DeleteConsumerAsync($"{ServiceName}_Commands", ServiceName, Ct); + await jetStream.DeleteConsumerAsync($"{ServiceName}_Commands", ServiceName, Ct); var error = await WaitForErrorAsync(errors); Assert.Equal(10014, Assert.IsType(error.InnerException).Error.ErrCode); @@ -172,8 +174,10 @@ public async Task ShouldReportAnErrorToSubscribersWhenTheCommandStreamIsDeleted( .HandleCommand((command, context, ct) => Task.CompletedTask)); var errors = RecordErrors(worker.Services.GetRequiredService()); await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); + var jetStream = new NatsJSContext(connection); + await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Commands", ServiceName); - await new NatsJSContext(connection).DeleteStreamAsync($"{ServiceName}_Commands", Ct); + await jetStream.DeleteStreamAsync($"{ServiceName}_Commands", Ct); var error = await WaitForErrorAsync(errors); Assert.Equal(10059, Assert.IsType(error.InnerException).Error.ErrCode); @@ -220,6 +224,14 @@ private static async Task ConsumerExistsAsync(INatsJSContext jetStream, st } } + // Deleting a consumer, or its stream, answers the pulls waiting on it. A pull still on its way is dropped without + // an answer, and the client gives up on it only when it misses two idle heartbeats, 30 seconds by default, so a + // test that deletes one waits until the bus's first pull is waiting. + private static async Task WaitUntilPullingAsync(INatsJSContext jetStream, string stream, string consumer) + { + await WaitUntilAsync(async () => (await jetStream.GetConsumerAsync(stream, consumer, Ct)).Info.NumWaiting > 0); + } + private static async Task WaitForErrorAsync(ConcurrentQueue errors) { await WaitUntilAsync(() => !errors.IsEmpty); From 16dc8e589431d2157d0a1d8be3ab8c0e91b86ec0 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:23:55 -1000 Subject: [PATCH 2/6] Give up on an unanswered JetStream pull after 10 seconds, not 30 nats-server drops a pull that reaches a consumer as it's deleted, without an answer, and the client gives up on a pull only after it misses two of the server's idle heartbeats: 30 seconds with the client's default for a 30 second pull. Until then the bus doesn't notice the consumer has gone. Pulls now ask for a heartbeat every 5 seconds, so the bus pulls again, and finds the consumer gone, after 10. Co-Authored-By: Claude Opus 5.5 --- .../TestJetStreamMessageSourceRetry.cs | 20 +++++++++++++++++++ .../JetStream/TestNatsJetStreamReconnect.cs | 4 ++-- Obvs.NATS/JetStream/MessageSource.cs | 5 +++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs b/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs index 79e90204..828a9a90 100644 --- a/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs +++ b/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs @@ -198,6 +198,26 @@ public async Task ShouldRecreateTheBusesOwnConsumerWhenItIsDeleted() .MustHaveHappenedOnceExactly(); } + [Fact] + public async Task ShouldGiveUpOnAPullThatGoesUnansweredWithinTenSeconds() + { + // The server drops a pull that reaches a consumer as it's deleted, without an answer. The client gives up on a + // pull when it misses two of the server's idle heartbeats, then the loop pulls again and finds the consumer gone. + var heartbeats = new ConcurrentQueue(); + _thenPull = (opts, cancellationToken) => + { + heartbeats.Enqueue(opts?.IdleHeartbeat); + return WaitUntilCancelledAsync(opts, cancellationToken); + }; + + using var subscription = Subscribe(Commands()); + + await WaitUntilAsync(() => !heartbeats.IsEmpty); + var heartbeat = heartbeats.Single(); + Assert.True(heartbeat > TimeSpan.Zero && heartbeat.Value.Ticks * 2 <= TimeSpan.FromSeconds(10).Ticks, + $"Pulls with an idle heartbeat of {heartbeat?.ToString() ?? "the client's default"}."); + } + [Fact] public async Task ShouldReportAnErrorWhenTheServerNoLongerHasJetStream() { diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs index a38b928e..60516b25 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs @@ -225,8 +225,8 @@ private static async Task ConsumerExistsAsync(INatsJSContext jetStream, st } // Deleting a consumer, or its stream, answers the pulls waiting on it. A pull still on its way is dropped without - // an answer, and the client gives up on it only when it misses two idle heartbeats, 30 seconds by default, so a - // test that deletes one waits until the bus's first pull is waiting. + // an answer, and the bus gives up on it only when it misses two idle heartbeats, 10 seconds, so a test that + // deletes one waits until the bus's first pull is waiting. private static async Task WaitUntilPullingAsync(INatsJSContext jetStream, string stream, string consumer) { await WaitUntilAsync(async () => (await jetStream.GetConsumerAsync(stream, consumer, Ct)).Info.NumWaiting > 0); diff --git a/Obvs.NATS/JetStream/MessageSource.cs b/Obvs.NATS/JetStream/MessageSource.cs index 16f2fbd0..6b57912d 100644 --- a/Obvs.NATS/JetStream/MessageSource.cs +++ b/Obvs.NATS/JetStream/MessageSource.cs @@ -28,6 +28,10 @@ internal sealed class MessageSource : IMessageSource private static readonly TimeSpan FirstRetryDelay = TimeSpan.FromMilliseconds(250); private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromSeconds(5); + // How often the server tells a waiting pull it's still there. The client gives up on a pull after missing two, as + // when the server dropped it on its way to a consumer being deleted: 30 seconds with the client's default. + private static readonly TimeSpan PullHeartbeat = TimeSpan.FromSeconds(5); + // JetStream's error codes for a consumer that doesn't exist, and for a server or account without JetStream private const int ConsumerNotFound = 10014; private const int JetStreamNotEnabledForAccount = 10039; @@ -368,6 +372,7 @@ private static async Task> PullAsync(INatsJSConsumer consumer var unanswered = false; var opts = new NatsJSNextOpts { + IdleHeartbeat = PullHeartbeat, NotificationHandler = (notification, cancellationToken) => { if (notification is NatsJSNoRespondersNotification) From b2fa7fd6e835e996490092362bb4e9c96a94f1e5 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:28:47 -0700 Subject: [PATCH 3/6] Wait for the bus's event pull, not 500ms, before deleting its consumer ShouldRecreateTheBusesOwnEventConsumerWhenItIsDeleted gave the bus 500ms to start pulling before deleting its event consumer, a guess that a slow machine could miss, and then the deletion could race the bus's first pull. It now waits until the consumer has a pull waiting, as the other deletion tests do. Co-Authored-By: Claude Opus 5.5 --- Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs index 60516b25..75e6d336 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs @@ -133,9 +133,9 @@ public async Task ShouldRecreateTheBusesOwnEventConsumerWhenItIsDeleted() var errors = RecordErrors(bus); var received = new ConcurrentQueue(); using var subscription = bus.Events.OfType().Subscribe(placed => received.Enqueue(placed.OrderId)); - await Task.Delay(500, Ct); await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); var jetStream = new NatsJSContext(connection); + await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Events", instanceId); await jetStream.DeleteConsumerAsync($"{ServiceName}_Events", instanceId, Ct); await WaitUntilAsync(async () => await ConsumerExistsAsync(jetStream, $"{ServiceName}_Events", instanceId)); From 6f43fc2d5050786df55fdd92ee254f766e55049d Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:33:45 -0700 Subject: [PATCH 4/6] Wait for the bus's event pull, not 500ms, before restarting the server ShouldKeepReceivingEventsAfterTheServerRestarts gave the bus 500ms to start pulling before killing the server. On a slow machine the restart could come first, and the test wouldn't check that a bus that was receiving keeps receiving. It now names the bus's instance, so it knows its event consumer, and waits until that consumer has a pull waiting. Co-Authored-By: Claude Opus 5.5 --- Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs index 75e6d336..f16e1204 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs @@ -59,12 +59,16 @@ public async Task ShouldKeepHandlingCommandsAfterTheServerRestarts() public async Task ShouldKeepReceivingEventsAfterTheServerRestarts() { await using var server = await RestartableJetStreamServer.StartAsync(); - using var subscriber = await StartHostAsync(server, service => service.AsClient()); + const string instanceId = "orders-worker-1"; + using var subscriber = await StartHostAsync(server, service => service.AsClient(), instanceId); var bus = subscriber.Services.GetRequiredService(); var errors = RecordErrors(bus); var received = new ConcurrentQueue(); using var subscription = bus.Events.OfType().Subscribe(placed => received.Enqueue(placed.OrderId)); - await Task.Delay(500, Ct); + await using (var connection = new NatsConnection(new NatsOpts { Url = server.Url })) + { + await WaitUntilPullingAsync(new NatsJSContext(connection), $"{ServiceName}_Events", instanceId); + } await server.RestartAsync(); var restarted = Stopwatch.StartNew(); From 4a242a1fcc1703930a8c744a88d54f06cdb6f69e Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:07:06 -0700 Subject: [PATCH 5/6] Wait for the bus's pulls, not 500ms, before stopping it in the hosting test ShouldNotReportAnErrorToSubscribersWhenTheBusStops gave the bus 500ms to start pulling before stopping it. On a slow machine the bus could stop first, and the test wouldn't check that stopping doesn't fail receive loops that are pulling. It now waits until its event and command consumers each have a pull waiting. WaitUntilPullingAsync moves to JetStreamServices so both test classes use it. Co-Authored-By: Claude Opus 5.5 --- Obvs.NATS.Tests/JetStream/JetStreamServices.cs | 17 +++++++++++++++++ .../JetStream/TestNatsJetStreamHosting.cs | 7 +++++-- .../JetStream/TestNatsJetStreamReconnect.cs | 8 -------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Obvs.NATS.Tests/JetStream/JetStreamServices.cs b/Obvs.NATS.Tests/JetStream/JetStreamServices.cs index c881e3a0..f40ff39b 100644 --- a/Obvs.NATS.Tests/JetStream/JetStreamServices.cs +++ b/Obvs.NATS.Tests/JetStream/JetStreamServices.cs @@ -60,6 +60,23 @@ public async Task MessagesOnStreamAsync(string stream) return info.Info.State.Messages; } + // Until a bus's first pull is waiting on a consumer, a test can't tell what deleting the consumer, or stopping the + // server or the bus, does to a bus that's receiving. And deleting a consumer only answers the pulls waiting on it: + // a pull still on its way is dropped without an answer, and the bus gives up on it only after missing two idle + // heartbeats, 10 seconds. + public static async Task WaitUntilPullingAsync(INatsJSContext jetStream, string stream, string consumer) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while ((await jetStream.GetConsumerAsync(stream, consumer, TestContext.Current.CancellationToken)).Info.NumWaiting == 0) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Nothing pulled from consumer {consumer} on stream {stream} in time."); + } + await Task.Delay(50, TestContext.Current.CancellationToken); + } + } + public async ValueTask DisposeAsync() { var streams = new List(); diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs index 6051131b..6f533492 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs @@ -108,12 +108,15 @@ public async Task ShouldDeleteTheBusesOwnConsumersWhenItStops() public async Task ShouldNotReportAnErrorToSubscribersWhenTheBusStops() { // Deleting this bus's consumers as it stops mustn't fail receive loops still subscribed - var host = await StartHostAsync(_services.Create("stop-quietly")); + var serviceName = _services.Create("stop-quietly"); + const string instanceId = "orders-worker-1"; + var host = await StartHostAsync(serviceName, instanceId: instanceId); var bus = host.Services.GetRequiredService(); var errors = new ConcurrentQueue(); var eventSubscription = bus.Events.Subscribe(_ => { }, errors.Enqueue); var commandSubscription = bus.Commands.Subscribe(_ => { }, errors.Enqueue); - await Task.Delay(500, Ct); + await WaitUntilPullingAsync(_services.JetStream, $"{serviceName}_Events", instanceId); + await WaitUntilPullingAsync(_services.JetStream, $"{serviceName}_Commands", serviceName); await host.StopAsync(Ct); host.Dispose(); diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs index f16e1204..1230cce6 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs @@ -228,14 +228,6 @@ private static async Task ConsumerExistsAsync(INatsJSContext jetStream, st } } - // Deleting a consumer, or its stream, answers the pulls waiting on it. A pull still on its way is dropped without - // an answer, and the bus gives up on it only when it misses two idle heartbeats, 10 seconds, so a test that - // deletes one waits until the bus's first pull is waiting. - private static async Task WaitUntilPullingAsync(INatsJSContext jetStream, string stream, string consumer) - { - await WaitUntilAsync(async () => (await jetStream.GetConsumerAsync(stream, consumer, Ct)).Info.NumWaiting > 0); - } - private static async Task WaitForErrorAsync(ConcurrentQueue errors) { await WaitUntilAsync(() => !errors.IsEmpty); From ac90651371a5d39904456a849b1a2743bf0df538 Mon Sep 17 00:00:00 2001 From: Christopher Read <1907605+christopherread@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:48:13 -0700 Subject: [PATCH 6/6] Start the tests' nats-servers on ports outgoing connections never get RestartableJetStreamServer took a free port from the OS, freed it, then started nats-server on it. The OS gives outgoing connections ports from the same range, so one could take the port first: nats-server then exited with 'address already in use', or the startup probe connected to itself and the server looked started when it wasn't. Servers now pick a port between 20000 and 32000, below the range the OS gives outgoing connections, and a first start that can't listen on its port picks another (servers starting at the same time can still clash). Restarts keep their port. A server is started once it logs that it's ready, not when something answers on its port, and a server that can't start says why, with the last lines it wrote. TestRestartableJetStreamServer starts 30 servers at a time, 10 times, while connections churn through ports. It failed 2 runs in 10 before and passed 30 in 30 after. It runs by itself, as its load held up another test's acknowledgements. Co-Authored-By: Claude Opus 5.5 --- .../JetStream/RestartableJetStreamServer.cs | 95 +++++++++++++------ .../TestRestartableJetStreamServer.cs | 83 ++++++++++++++++ 2 files changed, 151 insertions(+), 27 deletions(-) create mode 100644 Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs diff --git a/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs b/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs index 922daa3f..71842f8b 100644 --- a/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs +++ b/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs @@ -1,9 +1,8 @@ using System; +using System.Collections.Concurrent; using System.ComponentModel; using System.Diagnostics; using System.IO; -using System.Net; -using System.Net.Sockets; using System.Threading.Tasks; namespace Obvs.NATS.Tests.JetStream @@ -14,8 +13,15 @@ public sealed class RestartableJetStreamServer : IAsyncDisposable { private static readonly TimeSpan StartTimeout = TimeSpan.FromSeconds(10); - private readonly int _port = FreePort(); + // Below the ports the OS gives outgoing connections (from 32768 on Linux, 49152 on macOS and Windows), which could + // take a server's port while it's down, or before it first starts. Servers of tests running at the same time can + // still pick the same one, so a server that can't listen on its port picks another. + private const int FirstPort = 20000; + private const int LastPort = 32000; + private const int Attempts = 10; + private readonly string _storeDirectory = Path.Combine(Path.GetTempPath(), $"obvs-js-{Guid.NewGuid():N}"); + private int _port; private Process _process; private RestartableJetStreamServer() @@ -29,14 +35,24 @@ private RestartableJetStreamServer() public static async Task StartAsync() { var server = new RestartableJetStreamServer(); - await server.StartAgainAsync(); - return server; + for (var attempt = 1; ; attempt++) + { + server._port = Random.Shared.Next(FirstPort, LastPort); + try + { + await server.StartAgainAsync(); + return server; + } + catch (PortInUseException) when (attempt < Attempts) + { + } + } } // Without JetStream, the server is core NATS only public async Task StartAgainAsync(bool jetStream = true) { - var arguments = jetStream ? $"-js -sd \"{_storeDirectory}\" -p {_port}" : $"-p {_port}"; + var arguments = jetStream ? $"-js -sd \"{_storeDirectory}\" -a 127.0.0.1 -p {_port}" : $"-a 127.0.0.1 -p {_port}"; var startInfo = new ProcessStartInfo(Executable, arguments) { UseShellExecute = false, @@ -52,28 +68,49 @@ public async Task StartAgainAsync(bool jetStream = true) throw new InvalidOperationException( $"Couldn't run '{Executable}'. Install nats-server and put it on the PATH, or set NATS_SERVER to its path.", exception); } - _process.OutputDataReceived += (sender, args) => { }; - _process.ErrorDataReceived += (sender, args) => { }; - _process.BeginOutputReadLine(); - _process.BeginErrorReadLine(); - - var deadline = DateTime.UtcNow + StartTimeout; - while (true) + var output = new ConcurrentQueue(); + var ready = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + DataReceivedEventHandler record = (sender, args) => { - if (_process.HasExited) + if (args.Data == null) { - throw new InvalidOperationException($"nats-server {arguments} exited with code {_process.ExitCode}."); + // It's exited, and everything it wrote has been read + ready.TrySetResult(false); + return; } - try + Record(output, args.Data); + if (args.Data.Contains("Server is ready")) { - using var client = new TcpClient(); - await client.ConnectAsync(IPAddress.Loopback, _port); - return; + ready.TrySetResult(true); } - catch (SocketException) when (DateTime.UtcNow < deadline) + }; + _process.OutputDataReceived += record; + _process.ErrorDataReceived += record; + _process.BeginOutputReadLine(); + _process.BeginErrorReadLine(); + + bool started; + try + { + started = await ready.Task.WaitAsync(StartTimeout); + } + catch (TimeoutException) + { + await StopAsync(); + throw new InvalidOperationException( + $"nats-server {arguments} didn't start in {StartTimeout}: {string.Join(Environment.NewLine, output)}"); + } + if (!started) + { + // Waits until both its outputs have been read + _process.WaitForExit(); + var written = string.Join(Environment.NewLine, output); + await StopAsync(); + if (written.Contains("Error listening on port")) { - await Task.Delay(50); + throw new PortInUseException(); } + throw new InvalidOperationException($"nats-server {arguments} exited: {written}"); } } @@ -109,13 +146,17 @@ public async ValueTask DisposeAsync() } } - private static int FreePort() + // The last lines it wrote, which say why it couldn't start + private static void Record(ConcurrentQueue output, string line) + { + output.Enqueue(line); + while (output.Count > 20 && output.TryDequeue(out _)) + { + } + } + + private sealed class PortInUseException : Exception { - var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - listener.Stop(); - return port; } } } diff --git a/Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs b/Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs new file mode 100644 index 00000000..8b0e449d --- /dev/null +++ b/Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs @@ -0,0 +1,83 @@ +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using NATS.Client.Core; +using Xunit; + +namespace Obvs.NATS.Tests.JetStream +{ + // The tests' own nats-server, which test classes running at the same time each start. Runs by itself, as its load + // would hold up other tests' acknowledgements. + [Collection(nameof(TestRestartableJetStreamServer))] + public class TestRestartableJetStreamServer + { + [CollectionDefinition(nameof(TestRestartableJetStreamServer), DisableParallelization = true)] + public sealed class RunsAlone + { + } + + [Fact, Trait("Category", "Explicit")] + public async Task ShouldStartManyServersAtOnce() + { + // Other tests' connections are given ports from the range the OS gives outgoing connections + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var accepting = Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + using var accepted = await listener.AcceptTcpClientAsync(stop.Token); + } + }, stop.Token); + var connecting = Enumerable.Range(0, 16).Select(_ => Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + // Closed with a reset, so its port is free again at once, not kept for a while + using var client = new TcpClient { LingerState = new LingerOption(true, 0) }; + await client.ConnectAsync(IPAddress.Loopback, ((IPEndPoint)listener.LocalEndpoint).Port, stop.Token); + } + }, stop.Token)).ToArray(); + try + { + await StartManyServersAsync(); + } + finally + { + stop.Cancel(); + listener.Stop(); + try + { + await Task.WhenAll(connecting.Concat(new[] { accepting })); + } + catch (System.Exception) + { + } + } + } + + private static async Task StartManyServersAsync() + { + for (var round = 0; round < 10; round++) + { + var servers = await Task.WhenAll(Enumerable.Range(0, 30).Select(_ => Task.Run(RestartableJetStreamServer.StartAsync))); + try + { + Assert.Equal(servers.Length, servers.Select(server => server.Url).Distinct().Count()); + await Task.WhenAll(servers.Select(async server => + { + await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); + await connection.ConnectAsync(); + })); + } + finally + { + await Task.WhenAll(servers.Select(server => server.DisposeAsync().AsTask())); + } + } + } + } +}