diff --git a/Obvs.NATS.Tests/JetStream/JetStreamServices.cs b/Obvs.NATS.Tests/JetStream/JetStreamServices.cs index c881e3a..f40ff39 100644 --- a/Obvs.NATS.Tests/JetStream/JetStreamServices.cs +++ b/Obvs.NATS.Tests/JetStream/JetStreamServices.cs @@ -60,6 +60,23 @@ public async Task MessagesOnStreamAsync(string stream) return info.Info.State.Messages; } + // Until a bus's first pull is waiting on a consumer, a test can't tell what deleting the consumer, or stopping the + // server or the bus, does to a bus that's receiving. And deleting a consumer only answers the pulls waiting on it: + // a pull still on its way is dropped without an answer, and the bus gives up on it only after missing two idle + // heartbeats, 10 seconds. + public static async Task WaitUntilPullingAsync(INatsJSContext jetStream, string stream, string consumer) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while ((await jetStream.GetConsumerAsync(stream, consumer, TestContext.Current.CancellationToken)).Info.NumWaiting == 0) + { + if (DateTime.UtcNow > deadline) + { + throw new TimeoutException($"Nothing pulled from consumer {consumer} on stream {stream} in time."); + } + await Task.Delay(50, TestContext.Current.CancellationToken); + } + } + public async ValueTask DisposeAsync() { var streams = new List(); diff --git a/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs b/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs index 922daa3..71842f8 100644 --- a/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs +++ b/Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs @@ -1,9 +1,8 @@ using System; +using System.Collections.Concurrent; using System.ComponentModel; using System.Diagnostics; using System.IO; -using System.Net; -using System.Net.Sockets; using System.Threading.Tasks; namespace Obvs.NATS.Tests.JetStream @@ -14,8 +13,15 @@ public sealed class RestartableJetStreamServer : IAsyncDisposable { private static readonly TimeSpan StartTimeout = TimeSpan.FromSeconds(10); - private readonly int _port = FreePort(); + // Below the ports the OS gives outgoing connections (from 32768 on Linux, 49152 on macOS and Windows), which could + // take a server's port while it's down, or before it first starts. Servers of tests running at the same time can + // still pick the same one, so a server that can't listen on its port picks another. + private const int FirstPort = 20000; + private const int LastPort = 32000; + private const int Attempts = 10; + private readonly string _storeDirectory = Path.Combine(Path.GetTempPath(), $"obvs-js-{Guid.NewGuid():N}"); + private int _port; private Process _process; private RestartableJetStreamServer() @@ -29,14 +35,24 @@ private RestartableJetStreamServer() public static async Task StartAsync() { var server = new RestartableJetStreamServer(); - await server.StartAgainAsync(); - return server; + for (var attempt = 1; ; attempt++) + { + server._port = Random.Shared.Next(FirstPort, LastPort); + try + { + await server.StartAgainAsync(); + return server; + } + catch (PortInUseException) when (attempt < Attempts) + { + } + } } // Without JetStream, the server is core NATS only public async Task StartAgainAsync(bool jetStream = true) { - var arguments = jetStream ? $"-js -sd \"{_storeDirectory}\" -p {_port}" : $"-p {_port}"; + var arguments = jetStream ? $"-js -sd \"{_storeDirectory}\" -a 127.0.0.1 -p {_port}" : $"-a 127.0.0.1 -p {_port}"; var startInfo = new ProcessStartInfo(Executable, arguments) { UseShellExecute = false, @@ -52,28 +68,49 @@ public async Task StartAgainAsync(bool jetStream = true) throw new InvalidOperationException( $"Couldn't run '{Executable}'. Install nats-server and put it on the PATH, or set NATS_SERVER to its path.", exception); } - _process.OutputDataReceived += (sender, args) => { }; - _process.ErrorDataReceived += (sender, args) => { }; - _process.BeginOutputReadLine(); - _process.BeginErrorReadLine(); - - var deadline = DateTime.UtcNow + StartTimeout; - while (true) + var output = new ConcurrentQueue(); + var ready = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + DataReceivedEventHandler record = (sender, args) => { - if (_process.HasExited) + if (args.Data == null) { - throw new InvalidOperationException($"nats-server {arguments} exited with code {_process.ExitCode}."); + // It's exited, and everything it wrote has been read + ready.TrySetResult(false); + return; } - try + Record(output, args.Data); + if (args.Data.Contains("Server is ready")) { - using var client = new TcpClient(); - await client.ConnectAsync(IPAddress.Loopback, _port); - return; + ready.TrySetResult(true); } - catch (SocketException) when (DateTime.UtcNow < deadline) + }; + _process.OutputDataReceived += record; + _process.ErrorDataReceived += record; + _process.BeginOutputReadLine(); + _process.BeginErrorReadLine(); + + bool started; + try + { + started = await ready.Task.WaitAsync(StartTimeout); + } + catch (TimeoutException) + { + await StopAsync(); + throw new InvalidOperationException( + $"nats-server {arguments} didn't start in {StartTimeout}: {string.Join(Environment.NewLine, output)}"); + } + if (!started) + { + // Waits until both its outputs have been read + _process.WaitForExit(); + var written = string.Join(Environment.NewLine, output); + await StopAsync(); + if (written.Contains("Error listening on port")) { - await Task.Delay(50); + throw new PortInUseException(); } + throw new InvalidOperationException($"nats-server {arguments} exited: {written}"); } } @@ -109,13 +146,17 @@ public async ValueTask DisposeAsync() } } - private static int FreePort() + // The last lines it wrote, which say why it couldn't start + private static void Record(ConcurrentQueue output, string line) + { + output.Enqueue(line); + while (output.Count > 20 && output.TryDequeue(out _)) + { + } + } + + private sealed class PortInUseException : Exception { - var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - listener.Stop(); - return port; } } } diff --git a/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs b/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs index 79e9020..828a9a9 100644 --- a/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs +++ b/Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs @@ -198,6 +198,26 @@ public async Task ShouldRecreateTheBusesOwnConsumerWhenItIsDeleted() .MustHaveHappenedOnceExactly(); } + [Fact] + public async Task ShouldGiveUpOnAPullThatGoesUnansweredWithinTenSeconds() + { + // The server drops a pull that reaches a consumer as it's deleted, without an answer. The client gives up on a + // pull when it misses two of the server's idle heartbeats, then the loop pulls again and finds the consumer gone. + var heartbeats = new ConcurrentQueue(); + _thenPull = (opts, cancellationToken) => + { + heartbeats.Enqueue(opts?.IdleHeartbeat); + return WaitUntilCancelledAsync(opts, cancellationToken); + }; + + using var subscription = Subscribe(Commands()); + + await WaitUntilAsync(() => !heartbeats.IsEmpty); + var heartbeat = heartbeats.Single(); + Assert.True(heartbeat > TimeSpan.Zero && heartbeat.Value.Ticks * 2 <= TimeSpan.FromSeconds(10).Ticks, + $"Pulls with an idle heartbeat of {heartbeat?.ToString() ?? "the client's default"}."); + } + [Fact] public async Task ShouldReportAnErrorWhenTheServerNoLongerHasJetStream() { diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs index 6051131..6f53349 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs @@ -108,12 +108,15 @@ public async Task ShouldDeleteTheBusesOwnConsumersWhenItStops() public async Task ShouldNotReportAnErrorToSubscribersWhenTheBusStops() { // Deleting this bus's consumers as it stops mustn't fail receive loops still subscribed - var host = await StartHostAsync(_services.Create("stop-quietly")); + var serviceName = _services.Create("stop-quietly"); + const string instanceId = "orders-worker-1"; + var host = await StartHostAsync(serviceName, instanceId: instanceId); var bus = host.Services.GetRequiredService(); var errors = new ConcurrentQueue(); var eventSubscription = bus.Events.Subscribe(_ => { }, errors.Enqueue); var commandSubscription = bus.Commands.Subscribe(_ => { }, errors.Enqueue); - await Task.Delay(500, Ct); + await WaitUntilPullingAsync(_services.JetStream, $"{serviceName}_Events", instanceId); + await WaitUntilPullingAsync(_services.JetStream, $"{serviceName}_Commands", serviceName); await host.StopAsync(Ct); host.Dispose(); diff --git a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs index ce5bc9f..1230cce 100644 --- a/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs +++ b/Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs @@ -59,12 +59,16 @@ public async Task ShouldKeepHandlingCommandsAfterTheServerRestarts() public async Task ShouldKeepReceivingEventsAfterTheServerRestarts() { await using var server = await RestartableJetStreamServer.StartAsync(); - using var subscriber = await StartHostAsync(server, service => service.AsClient()); + const string instanceId = "orders-worker-1"; + using var subscriber = await StartHostAsync(server, service => service.AsClient(), instanceId); var bus = subscriber.Services.GetRequiredService(); var errors = RecordErrors(bus); var received = new ConcurrentQueue(); using var subscription = bus.Events.OfType().Subscribe(placed => received.Enqueue(placed.OrderId)); - await Task.Delay(500, Ct); + await using (var connection = new NatsConnection(new NatsOpts { Url = server.Url })) + { + await WaitUntilPullingAsync(new NatsJSContext(connection), $"{ServiceName}_Events", instanceId); + } await server.RestartAsync(); var restarted = Stopwatch.StartNew(); @@ -133,9 +137,9 @@ public async Task ShouldRecreateTheBusesOwnEventConsumerWhenItIsDeleted() var errors = RecordErrors(bus); var received = new ConcurrentQueue(); using var subscription = bus.Events.OfType().Subscribe(placed => received.Enqueue(placed.OrderId)); - await Task.Delay(500, Ct); await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); var jetStream = new NatsJSContext(connection); + await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Events", instanceId); await jetStream.DeleteConsumerAsync($"{ServiceName}_Events", instanceId, Ct); await WaitUntilAsync(async () => await ConsumerExistsAsync(jetStream, $"{ServiceName}_Events", instanceId)); @@ -157,8 +161,10 @@ public async Task ShouldReportAnErrorToSubscribersWhenTheSharedCommandConsumerIs .HandleCommand((command, context, ct) => Task.CompletedTask)); var errors = RecordErrors(worker.Services.GetRequiredService()); await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); + var jetStream = new NatsJSContext(connection); + await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Commands", ServiceName); - await new NatsJSContext(connection).DeleteConsumerAsync($"{ServiceName}_Commands", ServiceName, Ct); + await jetStream.DeleteConsumerAsync($"{ServiceName}_Commands", ServiceName, Ct); var error = await WaitForErrorAsync(errors); Assert.Equal(10014, Assert.IsType(error.InnerException).Error.ErrCode); @@ -172,8 +178,10 @@ public async Task ShouldReportAnErrorToSubscribersWhenTheCommandStreamIsDeleted( .HandleCommand((command, context, ct) => Task.CompletedTask)); var errors = RecordErrors(worker.Services.GetRequiredService()); await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); + var jetStream = new NatsJSContext(connection); + await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Commands", ServiceName); - await new NatsJSContext(connection).DeleteStreamAsync($"{ServiceName}_Commands", Ct); + await jetStream.DeleteStreamAsync($"{ServiceName}_Commands", Ct); var error = await WaitForErrorAsync(errors); Assert.Equal(10059, Assert.IsType(error.InnerException).Error.ErrCode); diff --git a/Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs b/Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs new file mode 100644 index 0000000..8b0e449 --- /dev/null +++ b/Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs @@ -0,0 +1,83 @@ +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using NATS.Client.Core; +using Xunit; + +namespace Obvs.NATS.Tests.JetStream +{ + // The tests' own nats-server, which test classes running at the same time each start. Runs by itself, as its load + // would hold up other tests' acknowledgements. + [Collection(nameof(TestRestartableJetStreamServer))] + public class TestRestartableJetStreamServer + { + [CollectionDefinition(nameof(TestRestartableJetStreamServer), DisableParallelization = true)] + public sealed class RunsAlone + { + } + + [Fact, Trait("Category", "Explicit")] + public async Task ShouldStartManyServersAtOnce() + { + // Other tests' connections are given ports from the range the OS gives outgoing connections + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var accepting = Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + using var accepted = await listener.AcceptTcpClientAsync(stop.Token); + } + }, stop.Token); + var connecting = Enumerable.Range(0, 16).Select(_ => Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + // Closed with a reset, so its port is free again at once, not kept for a while + using var client = new TcpClient { LingerState = new LingerOption(true, 0) }; + await client.ConnectAsync(IPAddress.Loopback, ((IPEndPoint)listener.LocalEndpoint).Port, stop.Token); + } + }, stop.Token)).ToArray(); + try + { + await StartManyServersAsync(); + } + finally + { + stop.Cancel(); + listener.Stop(); + try + { + await Task.WhenAll(connecting.Concat(new[] { accepting })); + } + catch (System.Exception) + { + } + } + } + + private static async Task StartManyServersAsync() + { + for (var round = 0; round < 10; round++) + { + var servers = await Task.WhenAll(Enumerable.Range(0, 30).Select(_ => Task.Run(RestartableJetStreamServer.StartAsync))); + try + { + Assert.Equal(servers.Length, servers.Select(server => server.Url).Distinct().Count()); + await Task.WhenAll(servers.Select(async server => + { + await using var connection = new NatsConnection(new NatsOpts { Url = server.Url }); + await connection.ConnectAsync(); + })); + } + finally + { + await Task.WhenAll(servers.Select(server => server.DisposeAsync().AsTask())); + } + } + } + } +} diff --git a/Obvs.NATS/JetStream/MessageSource.cs b/Obvs.NATS/JetStream/MessageSource.cs index 16f2fbd..6b57912 100644 --- a/Obvs.NATS/JetStream/MessageSource.cs +++ b/Obvs.NATS/JetStream/MessageSource.cs @@ -28,6 +28,10 @@ internal sealed class MessageSource : IMessageSource private static readonly TimeSpan FirstRetryDelay = TimeSpan.FromMilliseconds(250); private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromSeconds(5); + // How often the server tells a waiting pull it's still there. The client gives up on a pull after missing two, as + // when the server dropped it on its way to a consumer being deleted: 30 seconds with the client's default. + private static readonly TimeSpan PullHeartbeat = TimeSpan.FromSeconds(5); + // JetStream's error codes for a consumer that doesn't exist, and for a server or account without JetStream private const int ConsumerNotFound = 10014; private const int JetStreamNotEnabledForAccount = 10039; @@ -368,6 +372,7 @@ private static async Task> PullAsync(INatsJSConsumer consumer var unanswered = false; var opts = new NatsJSNextOpts { + IdleHeartbeat = PullHeartbeat, NotificationHandler = (notification, cancellationToken) => { if (notification is NatsJSNoRespondersNotification)