From 0cfbf6e49fd8ec67d2a9d9c8e4e0a0f43c2f7fb1 Mon Sep 17 00:00:00 2001
From: HeavenVR <git@heavenvr.tech>
Date: Fri, 13 Nov 2026 17:10:20 +0100
Subject: [PATCH] fix: drop hub control commands that waited too long to send

Control commands for a hub are queued on its websocket send channel. When
the hub is slow, that queue backs up, and everything in it was still
delivered later, including commands from an API token or session that had
been revoked, or a share that had been removed, in the meantime.

A control command now expires after 5 seconds in the send queue and is
dropped instead of written. Live control ticks are unaffected, they are
sent and awaited one at a time.
---
 Common/Constants/Constants.cs                 |  2 ++
 Common/Websocket/WebsockBaseController.cs     | 34 +++++++++++++++++--
 .../Controllers/HubV1Controller.cs            |  3 +-
 .../Controllers/HubV2Controller.cs            |  2 +-
 4 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/Common/Constants/Constants.cs b/Common/Constants/Constants.cs
index 659c9b30..d4ff3203 100644
--- a/Common/Constants/Constants.cs
+++ b/Common/Constants/Constants.cs
@@ -22,5 +22,7 @@ public static class Duration
     public static readonly TimeSpan DeviceKeepAliveInitialTimeout = TimeSpan.FromSeconds(65);
     public static readonly TimeSpan DeviceKeepAliveTimeout = TimeSpan.FromSeconds(35);
 
+    public static readonly TimeSpan HubControlMaxQueueAge = TimeSpan.FromSeconds(5);
+
     public static readonly TimeSpan DeactivatedAccountRetentionTime = TimeSpan.FromDays(14);
 }
diff --git a/Common/Websocket/WebsockBaseController.cs b/Common/Websocket/WebsockBaseController.cs
index 6d8fe2b6..3b3d22a7 100644
--- a/Common/Websocket/WebsockBaseController.cs
+++ b/Common/Websocket/WebsockBaseController.cs
@@ -1,4 +1,5 @@
-using System.Net.WebSockets;
+using System.Diagnostics;
+using System.Net.WebSockets;
 using System.Text.Json;
 using System.Threading.Channels;
 using Microsoft.AspNetCore.Mvc;
@@ -45,7 +46,12 @@ public abstract class WebsocketBaseController<T> : OpenShockControllerBase, IAsy
     /// <summary>
     /// A message waiting to be sent, <paramref name="Sent"/> is completed with whether it was written to the websocket
     /// </summary>
-    protected readonly record struct QueuedMessage(T Data, TaskCompletionSource<bool>? Sent);
+    /// <param name="Data"></param>
+    /// <param name="Sent"></param>
+    /// <param name="ExpiresAt">
+    /// <see cref="Stopwatch.GetTimestamp"/> after which the message is dropped instead of written, 0 for never.
+    /// </param>
+    protected readonly record struct QueuedMessage(T Data, TaskCompletionSource<bool>? Sent, long ExpiresAt = 0);
 
 #pragma warning disable IDISP008
     protected WebSocket? WebSocket;
@@ -73,6 +79,23 @@ public ValueTask QueueMessage(T data)
         return Channel.Writer.WriteAsync(new QueuedMessage(data, null), LinkedToken);
     }
 
+    /// <summary>
+    /// Queue a message that is only worth sending promptly. If the client is so far behind that it has
+    /// waited longer than <paramref name="maxQueueAge"/> when its turn comes, it is dropped.
+    /// </summary>
+    [NonAction]
+    public ValueTask QueueMessage(T data, TimeSpan maxQueueAge)
+    {
+        if (!CanSend())
+        {
+            Logger.LogDebug("WebSocket is null, abort or closed, not sending message");
+            return ValueTask.CompletedTask;
+        }
+
+        var expiresAt = Stopwatch.GetTimestamp() + (long)(maxQueueAge.TotalSeconds * Stopwatch.Frequency);
+        return Channel.Writer.WriteAsync(new QueuedMessage(data, null, expiresAt), LinkedToken);
+    }
+
     /// <summary>
     /// Queue a message and wait until it has been written to the websocket
     /// </summary>
@@ -207,6 +230,13 @@ private async Task MessageLoop()
         {
             await foreach (var msg in Channel.Reader.ReadAllAsync(LinkedToken))
             {
+                if (msg.ExpiresAt != 0 && Stopwatch.GetTimestamp() > msg.ExpiresAt)
+                {
+                    Logger.LogDebug("Dropping a message that waited too long in the send queue");
+                    msg.Sent?.TrySetResult(false);
+                    continue;
+                }
+
                 try
                 {
                     await SendWebSocketMessage(msg.Data, WebSocket!, LinkedToken);
diff --git a/LiveControlGateway/Controllers/HubV1Controller.cs b/LiveControlGateway/Controllers/HubV1Controller.cs
index 9430fa9a..2fd90185 100644
--- a/LiveControlGateway/Controllers/HubV1Controller.cs
+++ b/LiveControlGateway/Controllers/HubV1Controller.cs
@@ -3,6 +3,7 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.SignalR;
 using OpenShock.Common.Authentication;
+using OpenShock.Common.Constants;
 using OpenShock.Common.Hubs;
 using OpenShock.Common.Models;
 using OpenShock.Common.OpenShockDb;
@@ -172,7 +173,7 @@ await otaService.Error(CurrentHubId, payload.BootStatus.OtaUpdateId, false,
     /// <inheritdoc />
     [NonAction]
     public override ValueTask Control(IList<ShockerCommand> controlCommands)
-        => QueueMessage(ControlMessage(controlCommands));
+        => QueueMessage(ControlMessage(controlCommands), Duration.HubControlMaxQueueAge);
 
     /// <inheritdoc />
     [NonAction]
diff --git a/LiveControlGateway/Controllers/HubV2Controller.cs b/LiveControlGateway/Controllers/HubV2Controller.cs
index d3a12fa4..147ddbac 100644
--- a/LiveControlGateway/Controllers/HubV2Controller.cs
+++ b/LiveControlGateway/Controllers/HubV2Controller.cs
@@ -202,7 +202,7 @@ await otaService.Error(CurrentHubId, payload.BootStatus.OtaUpdateId, false,
 
     /// <inheritdoc />
     public override ValueTask Control(IList<ShockerCommand> controlCommands)
-        => QueueMessage(ControlMessage(controlCommands));
+        => QueueMessage(ControlMessage(controlCommands), Duration.HubControlMaxQueueAge);
 
     /// <inheritdoc />
     public override Task<bool> SendControl(IList<ShockerCommand> controlCommands)