Skip to content
Merged
17 changes: 17 additions & 0 deletions Obvs.NATS.Tests/JetStream/JetStreamServices.cs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,23 @@ public async Task<long> MessagesOnStreamAsync(string stream)
return info.Info.State.Messages;
}

// Until a bus's first pull is waiting on a consumer, a test can't tell what deleting the consumer, or stopping the
// server or the bus, does to a bus that's receiving. And deleting a consumer only answers the pulls waiting on it:
// a pull still on its way is dropped without an answer, and the bus gives up on it only after missing two idle
// heartbeats, 10 seconds.
public static async Task WaitUntilPullingAsync(INatsJSContext jetStream, string stream, string consumer)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while ((await jetStream.GetConsumerAsync(stream, consumer, TestContext.Current.CancellationToken)).Info.NumWaiting == 0)
{
if (DateTime.UtcNow > deadline)
{
throw new TimeoutException($"Nothing pulled from consumer {consumer} on stream {stream} in time.");
}
await Task.Delay(50, TestContext.Current.CancellationToken);
}
}

public async ValueTask DisposeAsync()
{
var streams = new List<string>();
Expand Down
95 changes: 68 additions & 27 deletions Obvs.NATS.Tests/JetStream/RestartableJetStreamServer.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
using System;
using System.Collections.Concurrent;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Threading.Tasks;

namespace Obvs.NATS.Tests.JetStream
Expand All @@ -14,8 +13,15 @@ public sealed class RestartableJetStreamServer : IAsyncDisposable
{
private static readonly TimeSpan StartTimeout = TimeSpan.FromSeconds(10);

private readonly int _port = FreePort();
// Below the ports the OS gives outgoing connections (from 32768 on Linux, 49152 on macOS and Windows), which could
// take a server's port while it's down, or before it first starts. Servers of tests running at the same time can
// still pick the same one, so a server that can't listen on its port picks another.
private const int FirstPort = 20000;
private const int LastPort = 32000;
private const int Attempts = 10;

private readonly string _storeDirectory = Path.Combine(Path.GetTempPath(), $"obvs-js-{Guid.NewGuid():N}");
private int _port;
private Process _process;

private RestartableJetStreamServer()
Expand All @@ -29,14 +35,24 @@ private RestartableJetStreamServer()
public static async Task<RestartableJetStreamServer> StartAsync()
{
var server = new RestartableJetStreamServer();
await server.StartAgainAsync();
return server;
for (var attempt = 1; ; attempt++)
{
server._port = Random.Shared.Next(FirstPort, LastPort);
try
{
await server.StartAgainAsync();
return server;
}
catch (PortInUseException) when (attempt < Attempts)
{
}
}
}

// Without JetStream, the server is core NATS only
public async Task StartAgainAsync(bool jetStream = true)
{
var arguments = jetStream ? $"-js -sd \"{_storeDirectory}\" -p {_port}" : $"-p {_port}";
var arguments = jetStream ? $"-js -sd \"{_storeDirectory}\" -a 127.0.0.1 -p {_port}" : $"-a 127.0.0.1 -p {_port}";
var startInfo = new ProcessStartInfo(Executable, arguments)
{
UseShellExecute = false,
Expand All @@ -52,28 +68,49 @@ public async Task StartAgainAsync(bool jetStream = true)
throw new InvalidOperationException(
$"Couldn't run '{Executable}'. Install nats-server and put it on the PATH, or set NATS_SERVER to its path.", exception);
}
_process.OutputDataReceived += (sender, args) => { };
_process.ErrorDataReceived += (sender, args) => { };
_process.BeginOutputReadLine();
_process.BeginErrorReadLine();

var deadline = DateTime.UtcNow + StartTimeout;
while (true)
var output = new ConcurrentQueue<string>();
var ready = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
DataReceivedEventHandler record = (sender, args) =>
{
if (_process.HasExited)
if (args.Data == null)
{
throw new InvalidOperationException($"nats-server {arguments} exited with code {_process.ExitCode}.");
// It's exited, and everything it wrote has been read
ready.TrySetResult(false);
return;
}
try
Record(output, args.Data);
if (args.Data.Contains("Server is ready"))
{
using var client = new TcpClient();
await client.ConnectAsync(IPAddress.Loopback, _port);
return;
ready.TrySetResult(true);
}
catch (SocketException) when (DateTime.UtcNow < deadline)
};
_process.OutputDataReceived += record;
_process.ErrorDataReceived += record;
_process.BeginOutputReadLine();
_process.BeginErrorReadLine();

bool started;
try
{
started = await ready.Task.WaitAsync(StartTimeout);
}
catch (TimeoutException)
{
await StopAsync();
throw new InvalidOperationException(
$"nats-server {arguments} didn't start in {StartTimeout}: {string.Join(Environment.NewLine, output)}");
}
if (!started)
{
// Waits until both its outputs have been read
_process.WaitForExit();
var written = string.Join(Environment.NewLine, output);
await StopAsync();
if (written.Contains("Error listening on port"))
{
await Task.Delay(50);
throw new PortInUseException();
}
throw new InvalidOperationException($"nats-server {arguments} exited: {written}");
}
}

Expand Down Expand Up @@ -109,13 +146,17 @@ public async ValueTask DisposeAsync()
}
}

private static int FreePort()
// The last lines it wrote, which say why it couldn't start
private static void Record(ConcurrentQueue<string> output, string line)
{
output.Enqueue(line);
while (output.Count > 20 && output.TryDequeue(out _))
{
}
}

private sealed class PortInUseException : Exception
{
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
listener.Stop();
return port;
}
}
}
20 changes: 20 additions & 0 deletions Obvs.NATS.Tests/JetStream/TestJetStreamMessageSourceRetry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,26 @@ public async Task ShouldRecreateTheBusesOwnConsumerWhenItIsDeleted()
.MustHaveHappenedOnceExactly();
}

[Fact]
public async Task ShouldGiveUpOnAPullThatGoesUnansweredWithinTenSeconds()
{
// The server drops a pull that reaches a consumer as it's deleted, without an answer. The client gives up on a
// pull when it misses two of the server's idle heartbeats, then the loop pulls again and finds the consumer gone.
var heartbeats = new ConcurrentQueue<TimeSpan?>();
_thenPull = (opts, cancellationToken) =>
{
heartbeats.Enqueue(opts?.IdleHeartbeat);
return WaitUntilCancelledAsync(opts, cancellationToken);
};

using var subscription = Subscribe(Commands());

await WaitUntilAsync(() => !heartbeats.IsEmpty);
var heartbeat = heartbeats.Single();
Assert.True(heartbeat > TimeSpan.Zero && heartbeat.Value.Ticks * 2 <= TimeSpan.FromSeconds(10).Ticks,
$"Pulls with an idle heartbeat of {heartbeat?.ToString() ?? "the client's default"}.");
}

[Fact]
public async Task ShouldReportAnErrorWhenTheServerNoLongerHasJetStream()
{
Expand Down
7 changes: 5 additions & 2 deletions Obvs.NATS.Tests/JetStream/TestNatsJetStreamHosting.cs
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,15 @@ public async Task ShouldDeleteTheBusesOwnConsumersWhenItStops()
public async Task ShouldNotReportAnErrorToSubscribersWhenTheBusStops()
{
// Deleting this bus's consumers as it stops mustn't fail receive loops still subscribed
var host = await StartHostAsync(_services.Create("stop-quietly"));
var serviceName = _services.Create("stop-quietly");
const string instanceId = "orders-worker-1";
var host = await StartHostAsync(serviceName, instanceId: instanceId);
var bus = host.Services.GetRequiredService<IServiceBus>();
var errors = new ConcurrentQueue<Exception>();
var eventSubscription = bus.Events.Subscribe(_ => { }, errors.Enqueue);
var commandSubscription = bus.Commands.Subscribe(_ => { }, errors.Enqueue);
await Task.Delay(500, Ct);
await WaitUntilPullingAsync(_services.JetStream, $"{serviceName}_Events", instanceId);
await WaitUntilPullingAsync(_services.JetStream, $"{serviceName}_Commands", serviceName);

await host.StopAsync(Ct);
host.Dispose();
Expand Down
18 changes: 13 additions & 5 deletions Obvs.NATS.Tests/JetStream/TestNatsJetStreamReconnect.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,16 @@ public async Task ShouldKeepHandlingCommandsAfterTheServerRestarts()
public async Task ShouldKeepReceivingEventsAfterTheServerRestarts()
{
await using var server = await RestartableJetStreamServer.StartAsync();
using var subscriber = await StartHostAsync(server, service => service.AsClient());
const string instanceId = "orders-worker-1";
using var subscriber = await StartHostAsync(server, service => service.AsClient(), instanceId);
var bus = subscriber.Services.GetRequiredService<IServiceBusClient>();
var errors = RecordErrors(bus);
var received = new ConcurrentQueue<string>();
using var subscription = bus.Events.OfType<OrderPlaced>().Subscribe(placed => received.Enqueue(placed.OrderId));
await Task.Delay(500, Ct);
await using (var connection = new NatsConnection(new NatsOpts { Url = server.Url }))
{
await WaitUntilPullingAsync(new NatsJSContext(connection), $"{ServiceName}_Events", instanceId);
}

await server.RestartAsync();
var restarted = Stopwatch.StartNew();
Expand Down Expand Up @@ -133,9 +137,9 @@ public async Task ShouldRecreateTheBusesOwnEventConsumerWhenItIsDeleted()
var errors = RecordErrors(bus);
var received = new ConcurrentQueue<string>();
using var subscription = bus.Events.OfType<OrderPlaced>().Subscribe(placed => received.Enqueue(placed.OrderId));
await Task.Delay(500, Ct);
await using var connection = new NatsConnection(new NatsOpts { Url = server.Url });
var jetStream = new NatsJSContext(connection);
await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Events", instanceId);

await jetStream.DeleteConsumerAsync($"{ServiceName}_Events", instanceId, Ct);
await WaitUntilAsync(async () => await ConsumerExistsAsync(jetStream, $"{ServiceName}_Events", instanceId));
Expand All @@ -157,8 +161,10 @@ public async Task ShouldReportAnErrorToSubscribersWhenTheSharedCommandConsumerIs
.HandleCommand<PlaceOrder>((command, context, ct) => Task.CompletedTask));
var errors = RecordErrors(worker.Services.GetRequiredService<IServiceBus>());
await using var connection = new NatsConnection(new NatsOpts { Url = server.Url });
var jetStream = new NatsJSContext(connection);
await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Commands", ServiceName);

await new NatsJSContext(connection).DeleteConsumerAsync($"{ServiceName}_Commands", ServiceName, Ct);
await jetStream.DeleteConsumerAsync($"{ServiceName}_Commands", ServiceName, Ct);

var error = await WaitForErrorAsync(errors);
Assert.Equal(10014, Assert.IsType<NatsJSApiException>(error.InnerException).Error.ErrCode);
Expand All @@ -172,8 +178,10 @@ public async Task ShouldReportAnErrorToSubscribersWhenTheCommandStreamIsDeleted(
.HandleCommand<PlaceOrder>((command, context, ct) => Task.CompletedTask));
var errors = RecordErrors(worker.Services.GetRequiredService<IServiceBus>());
await using var connection = new NatsConnection(new NatsOpts { Url = server.Url });
var jetStream = new NatsJSContext(connection);
await WaitUntilPullingAsync(jetStream, $"{ServiceName}_Commands", ServiceName);

await new NatsJSContext(connection).DeleteStreamAsync($"{ServiceName}_Commands", Ct);
await jetStream.DeleteStreamAsync($"{ServiceName}_Commands", Ct);

var error = await WaitForErrorAsync(errors);
Assert.Equal(10059, Assert.IsType<NatsJSApiException>(error.InnerException).Error.ErrCode);
Expand Down
83 changes: 83 additions & 0 deletions Obvs.NATS.Tests/JetStream/TestRestartableJetStreamServer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using NATS.Client.Core;
using Xunit;

namespace Obvs.NATS.Tests.JetStream
{
// The tests' own nats-server, which test classes running at the same time each start. Runs by itself, as its load
// would hold up other tests' acknowledgements.
[Collection(nameof(TestRestartableJetStreamServer))]
public class TestRestartableJetStreamServer
{
[CollectionDefinition(nameof(TestRestartableJetStreamServer), DisableParallelization = true)]
public sealed class RunsAlone
{
}

[Fact, Trait("Category", "Explicit")]
public async Task ShouldStartManyServersAtOnce()
{
// Other tests' connections are given ports from the range the OS gives outgoing connections
using var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
using var stop = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
var accepting = Task.Run(async () =>
{
while (!stop.IsCancellationRequested)
{
using var accepted = await listener.AcceptTcpClientAsync(stop.Token);
}
}, stop.Token);
var connecting = Enumerable.Range(0, 16).Select(_ => Task.Run(async () =>
{
while (!stop.IsCancellationRequested)
{
// Closed with a reset, so its port is free again at once, not kept for a while
using var client = new TcpClient { LingerState = new LingerOption(true, 0) };
await client.ConnectAsync(IPAddress.Loopback, ((IPEndPoint)listener.LocalEndpoint).Port, stop.Token);
}
}, stop.Token)).ToArray();
try
{
await StartManyServersAsync();
}
finally
{
stop.Cancel();
listener.Stop();
try
{
await Task.WhenAll(connecting.Concat(new[] { accepting }));
}
catch (System.Exception)
{
}
}
}

private static async Task StartManyServersAsync()
{
for (var round = 0; round < 10; round++)
{
var servers = await Task.WhenAll(Enumerable.Range(0, 30).Select(_ => Task.Run(RestartableJetStreamServer.StartAsync)));
try
{
Assert.Equal(servers.Length, servers.Select(server => server.Url).Distinct().Count());
await Task.WhenAll(servers.Select(async server =>
{
await using var connection = new NatsConnection(new NatsOpts { Url = server.Url });
await connection.ConnectAsync();
}));
}
finally
{
await Task.WhenAll(servers.Select(server => server.DisposeAsync().AsTask()));
}
}
}
}
}
5 changes: 5 additions & 0 deletions Obvs.NATS/JetStream/MessageSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ internal sealed class MessageSource<TMessage> : IMessageSource<TMessage>
private static readonly TimeSpan FirstRetryDelay = TimeSpan.FromMilliseconds(250);
private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromSeconds(5);

// How often the server tells a waiting pull it's still there. The client gives up on a pull after missing two, as
// when the server dropped it on its way to a consumer being deleted: 30 seconds with the client's default.
private static readonly TimeSpan PullHeartbeat = TimeSpan.FromSeconds(5);

// JetStream's error codes for a consumer that doesn't exist, and for a server or account without JetStream
private const int ConsumerNotFound = 10014;
private const int JetStreamNotEnabledForAccount = 10039;
Expand Down Expand Up @@ -368,6 +372,7 @@ private static async Task<INatsJSMsg<byte[]>> PullAsync(INatsJSConsumer consumer
var unanswered = false;
var opts = new NatsJSNextOpts
{
IdleHeartbeat = PullHeartbeat,
NotificationHandler = (notification, cancellationToken) =>
{
if (notification is NatsJSNoRespondersNotification)
Expand Down
Loading