Fix: x2 interpolation causes input lag issue

app/src/main/cpp/winlator/vk/framegen/fg_present.c

@@ -134,6 +134,10 @@ struct FgPresenter {
     uint64_t log_real;
     uint64_t log_generated;
     uint64_t acquire_misses;
+    // Present-queue model (see fg_record_and_present): when the last presented image would be
+    // shown, and the generated frames left out to drain the queue.
+    uint64_t queue_end_ns;
+    uint64_t trimmed;
 };
 
 static void fg_reset_swapchain(FgPresenter* fg);
@@ -804,6 +808,12 @@ static void fg_renew_semaphore(FgPresenter* fg, VkSemaphore* handle) {
     *handle = fresh;
 }
 
+static uint64_t fg_now_ns(void) {
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec;
+}
+
 static void fg_record_and_present(FgPresenter* fg, FgImport* source, AImage* image) {
     FgFrame* f = &fg->frames[fg->frame_index];
 
@@ -822,6 +832,7 @@ static void fg_record_and_present(FgPresenter* fg, FgImport* source, AImage* ima
     }
 
     uint32_t image_index = 0;
+    const uint64_t acquire_begin_ns = fg_now_ns();
     VkResult acq = vkAcquireNextImageKHR(fg->device, fg->swapchain, UINT64_MAX, f->acquire,
                                          VK_NULL_HANDLE, &image_index);
     if (acq != VK_SUCCESS && acq != VK_SUBOPTIMAL_KHR) {
@@ -830,6 +841,24 @@ static void fg_record_and_present(FgPresenter* fg, FgImport* source, AImage* ima
         return;
     }
 
+    // Keep the present queue short (see vk_renderer.c): FIFO shows one image per refresh and
+    // only the swapchain size bounds the queue, so output even slightly faster than the display
+    // fills it and every frame reaches the screen up to ~80 ms late. While more than two images
+    // wait beyond the next refresh, one generated frame per pair is left out until it drains.
+    const uint64_t period_ns = fg->refresh_rate > 1.0f ? (uint64_t)(1.0e9f / fg->refresh_rate) : 0;
+    if (period_ns > 0) {
+        const uint64_t now_ns = fg_now_ns();
+        if (now_ns - acquire_begin_ns > period_ns / 2 && fg->queue_end_ns < now_ns + 2 * period_ns) {
+            fg->queue_end_ns = now_ns + 2 * period_ns;
+        }
+        const uint64_t ahead = fg->queue_end_ns > now_ns + period_ns
+            ? (fg->queue_end_ns - now_ns - period_ns) / period_ns : 0;
+        if (ahead > 2 && planned > 0) {
+            planned--;
+            fg->trimmed++;
+        }
+    }
+
     uint64_t gen_timeout = FG_ACQUIRE_TIMEOUT_MAX_NS;
     if (fg->refresh_rate > 1.0f) {
         gen_timeout = (uint64_t)(2000000000.0f / fg->refresh_rate);
@@ -983,19 +1012,29 @@ static void fg_record_and_present(FgPresenter* fg, FgImport* source, AImage* ima
     VkResult pr = vkQueuePresentKHR(fg->queue, &pi);
     if (pr == VK_ERROR_OUT_OF_DATE_KHR) out_of_date = true;
 
+    // The queue model: each image presented leaves the queue one refresh after the one before.
+    if (period_ns > 0) {
+        const uint64_t now_ns = fg_now_ns();
+        for (uint32_t i = 0; i < gen_count + 1; i++) {
+            const uint64_t next = fg->queue_end_ns + period_ns;
+            fg->queue_end_ns = next > now_ns + period_ns ? next : now_ns + period_ns;
+        }
+    }
+
     fg->real_frames++;
     fg->generated_frames += gen_count;
     if ((fg->real_frames % FG_TELEMETRY_FRAMES) == 0) {
         const uint64_t d_real = fg->real_frames - fg->log_real;
         const uint64_t d_gen = fg->generated_frames - fg->log_generated;
         fg->log_real = fg->real_frames;
         fg->log_generated = fg->generated_frames;
-        FG_LOGI("framegen[%s] real=%llu made=%llu ratio=%.2f planned=%u got=%u images=%u div=%u misses=%llu",
+        FG_LOGI("framegen[%s] real=%llu made=%llu ratio=%.2f planned=%u got=%u images=%u div=%u "
+                "misses=%llu trimmed=%llu",
                 fg_engine_name(fg->active_engine),
                 (unsigned long long)fg->real_frames, (unsigned long long)fg->generated_frames,
                 d_real ? (double)(d_real + d_gen) / (double)d_real : 0.0, planned, gen_count,
                 fg->swapchain_image_count, fg->source_divisor,
-                (unsigned long long)fg->acquire_misses);
+                (unsigned long long)fg->acquire_misses, (unsigned long long)fg->trimmed);
     }
 
     fg->frame_index = (fg->frame_index + 1) % FG_FRAMES_IN_FLIGHT;

app/src/main/cpp/winlator/vk/vk_renderer.c

@@ -1901,6 +1901,9 @@ static void destroy_lsfg(VkRenderer* r) {
     r->framegen_real_frames = 0;
     r->framegen_made_frames = 0;
     r->framegen_draw_ns = 0;
+    r->framegen_queue_end_ns = 0;
+    r->framegen_trimmed = 0;
+    r->framegen_same_source = 0;
     r->framegen_gap_ns = 0;
     r->framegen_last_end_ns = 0;
     r->framegen_timed_frames = 0;
@@ -1927,6 +1930,9 @@ static void destroy_dis(VkRenderer* r) {
     r->framegen_real_frames = 0;
     r->framegen_made_frames = 0;
     r->framegen_draw_ns = 0;
+    r->framegen_queue_end_ns = 0;
+    r->framegen_trimmed = 0;
+    r->framegen_same_source = 0;
     r->framegen_gap_ns = 0;
     r->framegen_last_end_ns = 0;
     r->framegen_timed_frames = 0;
@@ -2709,8 +2715,10 @@ static bool record_and_submit_frame(VkRenderer* r) {
     VkCompositeTarget* composite = via_composite ? &r->composite[r->frame_index] : NULL;
 
     uint32_t image_index = 0;
+    const uint64_t acquire_begin_ns = vkr_monotonic_ns();
     VkResult acq = vkAcquireNextImageKHR(r->device, r->swapchain, UINT64_MAX,
                                          f->image_available, VK_NULL_HANDLE, &image_index);
+    const uint64_t real_acquire_wait_ns = vkr_monotonic_ns() - acquire_begin_ns;
     bool recreate_after_present = false;
     if (acq == VK_ERROR_OUT_OF_DATE_KHR) {
         r->surface_ready = false;
@@ -2741,6 +2749,42 @@ static bool record_and_submit_frame(VkRenderer* r) {
         }
     }
 
+    // Keep the present queue short. FIFO shows one image per refresh and nothing else bounds
+    // the queue but the swapchain's dozen images, so a game even slightly faster than the
+    // output cadence (60.5 fps doubled on 120 Hz), a draw without a new game frame, or a burst
+    // after a stall fills it, and from then on every frame - the game's own included - reaches
+    // the display up to ~80 ms late: input lag that a game held exactly at 30 fps never shows.
+    // A draw without a new game frame generates nothing (there is no new motion to show), and
+    // while more than two images wait beyond the next refresh one generated frame per pair is
+    // left out until the queue drains - one at a time, so the motion cadence stays even.
+    const uint64_t source_frames = __atomic_load_n(&r->framegen_source_frames, __ATOMIC_RELAXED);
+    const bool new_source_frame = source_frames != r->framegen_seen_source;
+    r->framegen_seen_source = source_frames;
+    const uint64_t refresh_period_ns =
+        r->framegen_refresh_rate > 1.0f ? (uint64_t)(1.0e9f / r->framegen_refresh_rate) : 0;
+    if (framegen_planned > 0 && !new_source_frame) {
+        framegen_planned = 0;
+        r->framegen_same_source++;
+    }
+    if (refresh_period_ns > 0) {
+        const uint64_t now_ns = vkr_monotonic_ns();
+        // A real acquire that blocked means the queue is at least that full, whatever the model.
+        if (real_acquire_wait_ns > refresh_period_ns / 2 &&
+            r->framegen_queue_end_ns < now_ns + 2 * refresh_period_ns) {
+            r->framegen_queue_end_ns = now_ns + 2 * refresh_period_ns;
+        }
+        const uint64_t ahead = r->framegen_queue_end_ns > now_ns + refresh_period_ns
+            ? (r->framegen_queue_end_ns - now_ns - refresh_period_ns) / refresh_period_ns : 0;
+        if (framegen_planned > 0) {
+            r->framegen_ahead_sum += ahead;
+            r->framegen_ahead_n++;
+            if (ahead > 2) {
+                framegen_planned--;
+                r->framegen_trimmed++;
+            }
+        }
+    }
+
     uint32_t gen_count = 0;
     uint32_t gen_image_index[VK_FRAMEGEN_MAX_GENERATIONS] = {0};
     for (uint32_t g = 0; g < framegen_planned; g++) {
@@ -2951,15 +2995,22 @@ static bool record_and_submit_frame(VkRenderer* r) {
                 r->framegen_gap_ns = 0;
                 r->framegen_timed_frames = 0;
                 VK_LOGI("framegen delivered real=%llu made=%llu ratio=%.2f planned=%u got=%u "
-                        "misses=%llu timeout=%.1fms images=%u capacity=%u draw=%.2fms gap=%.2fms",
+                        "misses=%llu timeout=%.1fms images=%u capacity=%u draw=%.2fms gap=%.2fms "
+                        "queue=%.2f trimmed=%llu same=%llu",
                         (unsigned long long)r->framegen_real_frames,
                         (unsigned long long)r->framegen_made_frames,
                         d_real ? (double)d_made / (double)d_real : 0.0,
                         framegen_planned, gen_count,
                         (unsigned long long)r->framegen_acquire_misses,
                         (double)gen_acquire_timeout / 1000000.0,
                         r->swapchain_image_count, framegen_capacity,
-                        draw_ms, gap_ms);
+                        draw_ms, gap_ms,
+                        r->framegen_ahead_n ? (double)r->framegen_ahead_sum /
+                                                  (double)r->framegen_ahead_n : 0.0,
+                        (unsigned long long)r->framegen_trimmed,
+                        (unsigned long long)r->framegen_same_source);
+                r->framegen_ahead_sum = 0;
+                r->framegen_ahead_n = 0;
             }
         }
 
@@ -3126,6 +3177,7 @@ static bool record_and_submit_frame(VkRenderer* r) {
     pi.pImageIndices = &image_index;
 
     bool gen_present_out_of_date = false;
+    uint32_t presented_now = 0;
     pthread_mutex_lock(&r->queue_mutex);
     for (uint32_t g = 0; g < gen_count; g++) {
         VkPresentInfoKHR gpi = {VK_STRUCTURE_TYPE_PRESENT_INFO_KHR};
@@ -3143,11 +3195,22 @@ static bool record_and_submit_frame(VkRenderer* r) {
             }
         } else {
             __atomic_fetch_add(&r->presented_frames, 1, __ATOMIC_RELAXED);
+            presented_now++;
         }
     }
     VkResult pr = vkQueuePresentKHR(r->graphics_queue, &pi);
     if (pr == VK_SUCCESS || pr == VK_SUBOPTIMAL_KHR) {
         __atomic_fetch_add(&r->presented_frames, 1, __ATOMIC_RELAXED);
+        presented_now++;
+    }
+    // The queue model: each image presented leaves the queue one refresh after the one before.
+    if (refresh_period_ns > 0) {
+        const uint64_t now_ns = vkr_monotonic_ns();
+        for (uint32_t i = 0; i < presented_now; i++) {
+            const uint64_t next = r->framegen_queue_end_ns + refresh_period_ns;
+            r->framegen_queue_end_ns = next > now_ns + refresh_period_ns ? next
+                                                                          : now_ns + refresh_period_ns;
+        }
     }
     // Present the mirror separately so its result doesn't disturb the display recreate logic below.
     if (rec_this_frame) {

app/src/main/cpp/winlator/vk/vk_state.h

@@ -443,6 +443,14 @@ typedef struct VkRenderer {
     uint64_t          framegen_log_made;
     uint64_t          framegen_made_frames;
     uint64_t          framegen_acquire_misses;
+    // Present-queue model for frame generation: FIFO shows one image per refresh, so the queue
+    // drains at the display rate while each draw adds its real frame and its generated ones.
+    uint64_t          framegen_queue_end_ns;   // when the last presented image would be shown
+    uint64_t          framegen_seen_source;    // game frame count at the last draw
+    uint64_t          framegen_trimmed;        // generated frames left out to drain the queue
+    uint64_t          framegen_same_source;    // draws without a new game frame
+    uint64_t          framegen_ahead_sum;
+    uint64_t          framegen_ahead_n;
     uint64_t          framegen_draw_ns;
     uint64_t          framegen_gap_ns;
     uint64_t          framegen_last_end_ns;