Fix Vulkan renderer

samples/Demos/01-Renderer/ElementalArt.c

@@ -21,8 +21,8 @@ void ElemArtInit(ElemGraphicsDevice graphicsDevice, ElemArtData* elemArtData)
     elemArtData->TextBufferCount = 0;
 
     elemArtData->MaxDraw2DCommandCount = 1024;
-    elemArtData->Draw2DCommands = (Draw2DCommand*)malloc(elemArtData->MaxDraw2DCommandCount);
-    elemArtData->Draw2DCommandsBuffer = SampleCreateGpuBuffer(&elemArtData->GpuMemory, elemArtData->MaxDraw2DCommandCount, ElemGraphicsResourceUsage_Read, "Draw2DCommandsBuffer");
+    elemArtData->Draw2DCommands = (Draw2DCommand*)malloc(elemArtData->MaxDraw2DCommandCount * sizeof(Draw2DCommand));
+    elemArtData->Draw2DCommandsBuffer = SampleCreateGpuBuffer(&elemArtData->GpuMemory, elemArtData->MaxDraw2DCommandCount * sizeof(Draw2DCommand), ElemGraphicsResourceUsage_Read, "Draw2DCommandsBuffer");
     elemArtData->Draw2DCommandCount = 0;
 }
 
@@ -66,7 +66,7 @@ void ElemArtRender(ElemCommandList commandList, ElemVector2 renderTargetSize, El
         .RenderTargetSize = renderTargetSize,
     };
 
-    ElemPushPipelineStateConstants(commandList, 0, (ElemDataSpan) { .Items = (uint8_t*)&parameters, .Length = sizeof(RaytracingShaderParameters) });
+    ElemPushPipelineStateConstants(commandList, 0, (ElemDataSpan) { .Items = (uint8_t*)&parameters, .Length = sizeof(DrawTextShaderParameters) });
 
     ElemDispatchMesh(commandList, 1, 1, 1);
     elemArtData->TextBufferCount = 0;

samples/Demos/01-Renderer/main.c

@@ -196,7 +196,7 @@ void InitSample(void* payload)
     ElemSetGraphicsOptions(&(ElemGraphicsOptions) { 
         .EnableDebugLayer = applicationPayload->AppSettings.GpuDebug, 
         .EnableGpuValidation = false, 
-        .EnableDebugBarrierInfo = false, 
+        .EnableDebugBarrierInfo = false,
         .EnableDebugStablePowerState = true,
         .PreferVulkan = applicationPayload->AppSettings.PreferVulkan 
     });
@@ -264,6 +264,9 @@ void InitSample(void* payload)
             .BlendOperation = ElemGraphicsBlendOperation_Add,
             .SourceBlendFactor = ElemGraphicsBlendFactor_SourceAlpha,
             .DestinationBlendFactor = ElemGraphicsBlendFactor_InverseSourceAlpha,
+            .BlendOperationAlpha = ElemGraphicsBlendOperation_Add,
+            .SourceBlendFactorAlpha = ElemGraphicsBlendFactor_Zero,
+            .DestinationBlendFactorAlpha = ElemGraphicsBlendFactor_One,
         }}, .Length = 1 },
     });
 
@@ -277,6 +280,9 @@ void InitSample(void* payload)
             .BlendOperation = ElemGraphicsBlendOperation_Add,
             .SourceBlendFactor = ElemGraphicsBlendFactor_SourceAlpha,
             .DestinationBlendFactor = ElemGraphicsBlendFactor_InverseSourceAlpha,
+            .BlendOperationAlpha = ElemGraphicsBlendOperation_Add,
+            .SourceBlendFactorAlpha = ElemGraphicsBlendFactor_Zero,
+            .DestinationBlendFactorAlpha = ElemGraphicsBlendFactor_One,
         }}, .Length = 1 },
     });
 
@@ -597,7 +603,6 @@ void UpdateSwapChain(const ElemSwapChainUpdateParameters* updateParameters, void
 
     ElemGraphicsResourceBarrier(commandList, applicationPayload->RenderTargetTextureReadDescriptor, NULL);
     ElemGraphicsResourceBarrier(commandList, applicationPayload->DebugUIData.UIRenderTargetTextureReadDescriptor, NULL);
-
     // TODO: Refactor that
     ElemInsertGraphicsTimestamp(commandList, applicationPayload->GpuTimestampData.TonemapTimestamp.StartGpuTimestamp);
     applicationPayload->GpuTimestampData.TonemapTimestamp.IsActive = true;
@@ -687,4 +692,4 @@ int main(int argc, const char* argv[])
         .Payload = &payload
     });
 
-}
+}

src/Elemental/Common/Graphics/CommandAllocatorPool.h

@@ -49,4 +49,4 @@ template<typename TCommandList>
 void ReleaseCommandListPoolItem(CommandListPoolItem<TCommandList>* commandListPoolItem);
 
 template<typename TCommandAllocator, typename TCommandList>
-void UpdateCommandAllocatorPoolItemFence(CommandAllocatorPoolItem<TCommandAllocator, TCommandList>* commandAllocatorPoolItem, ElemFence fence);
+void UpdateCommandAllocatorPoolItemFence(CommandAllocatorPoolItem<TCommandAllocator, TCommandList>* commandAllocatorPoolItem, ElemFence fence);

src/Elemental/Common/Graphics/Vulkan/VulkanCommandList.cpp

@@ -121,11 +121,6 @@ ElemCommandQueue VulkanCreateCommandQueue(ElemGraphicsDevice graphicsDevice, Ele
     VkSemaphore fence;
     AssertIfFailed(vkCreateSemaphore(graphicsDeviceData->Device, &createInfo, NULL, &fence));
 
-    createInfo = { VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO };
-
-    VkSemaphore presentSemaphore;
-    AssertIfFailed(vkCreateSemaphore(graphicsDeviceData->Device, &createInfo, NULL, &presentSemaphore));
-
     if (VulkanDebugLayerEnabled && options && options->DebugName)
     {
         VkDebugUtilsObjectNameInfoEXT nameInfo = { VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT };
@@ -151,7 +146,8 @@ ElemCommandQueue VulkanCreateCommandQueue(ElemGraphicsDevice graphicsDevice, Ele
         .GraphicsDevice = graphicsDevice,
         .Fence = fence,
         .FenceValue = 0,
-        .PresentSemaphore = presentSemaphore,
+        .AcquireSemaphore = VK_NULL_HANDLE,
+        .PresentSemaphore = VK_NULL_HANDLE,
         .LastCompletedFenceValue = 0,
         .CommandQueueFrequency = queueFrequency
     }); 
@@ -190,7 +186,6 @@ void VulkanFreeCommandQueue(ElemCommandQueue commandQueue)
     }
     // TODO: Free allocators and command buffers
 
-    vkDestroySemaphore(graphicsDeviceData->Device, commandQueueData->PresentSemaphore, nullptr);
     vkDestroySemaphore(graphicsDeviceData->Device, commandQueueData->Fence, nullptr);
 
     auto graphicsIdUnpacked = UnpackSystemDataPoolHandle(commandQueueData->GraphicsDevice);
@@ -369,10 +364,10 @@ void VulkanCommitCommandList(ElemCommandList commandList)
         vkCmdCopyQueryPoolResults(commandListData->DeviceObject, 
                                   graphicsDeviceData->QueryHeap.Storage->QueryHeap, 
                                   index, 
-                                  count, 
+                                  count,
                                   graphicsDeviceData->QueryHeap.Storage->QueryHeapReadbackBuffer.Buffer, 
                                   index * sizeof(uint64_t), 
-                                  sizeof(uint64_t), 
+                                  sizeof(uint64_t),
                                   VK_QUERY_RESULT_64_BIT);// | VK_QUERY_RESULT_WAIT_BIT);*/
 
         //CreateVulkanGraphicsBufferBarrier(commandListData->DeviceObject, graphicsDeviceData->QueryHeap.Storage->QueryHeapReadbackBuffer.Buffer, false);
@@ -405,17 +400,21 @@ ElemFence VulkanExecuteCommandLists(ElemCommandQueue commandQueue, ElemCommandLi
     auto commandQueueData = GetVulkanCommandQueueData(commandQueue);
     SystemAssert(commandQueueData);
 
+    auto fencesToWaitCount = options ? options->FencesToWait.Length : 0;
+    auto hasAcquireSemaphore = commandQueueData->AcquireSemaphore != VK_NULL_HANDLE;
+    auto waitSemaphoreCount = fencesToWaitCount + (hasAcquireSemaphore ? 1 : 0);
+
     Span<VkPipelineStageFlags> submitStageMasks = {};
     Span<VkSemaphore> waitSemaphores = {};
     Span<uint64_t> waitSemaphoreValues = {};
 
-    if (options && options->FencesToWait.Length > 0)
+    if (waitSemaphoreCount > 0)
     {
-        submitStageMasks = SystemPushArray<VkPipelineStageFlags>(stackMemoryArena, options->FencesToWait.Length);
-        waitSemaphores = SystemPushArray<VkSemaphore>(stackMemoryArena, options->FencesToWait.Length);
-        waitSemaphoreValues = SystemPushArray<uint64_t>(stackMemoryArena, options->FencesToWait.Length);
+        submitStageMasks = SystemPushArray<VkPipelineStageFlags>(stackMemoryArena, waitSemaphoreCount);
+        waitSemaphores = SystemPushArray<VkSemaphore>(stackMemoryArena, waitSemaphoreCount);
+        waitSemaphoreValues = SystemPushArray<uint64_t>(stackMemoryArena, waitSemaphoreCount);
 
-        for (uint32_t i = 0; i < options->FencesToWait.Length; i++)
+        for (uint32_t i = 0; i < fencesToWaitCount; i++)
         {
             auto fenceToWait = options->FencesToWait.Items[i];
 
@@ -431,6 +430,14 @@ ElemFence VulkanExecuteCommandLists(ElemCommandQueue commandQueue, ElemCommandLi
                 SystemLogDebugMessage(ElemLogMessageCategory_Graphics, "Waiting for fence before ExecuteCommandLists. (CommandQueue=%d, Value=%d)", fenceToWait.CommandQueue, fenceToWait.FenceValue);
             }
         }
+
+        if (hasAcquireSemaphore)
+        {
+            auto acquireSemaphoreIndex = fencesToWaitCount;
+            submitStageMasks[acquireSemaphoreIndex] = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
+            waitSemaphores[acquireSemaphoreIndex] = commandQueueData->AcquireSemaphore;
+            waitSemaphoreValues[acquireSemaphoreIndex] = 0;
+        }
     }
 
     bool hasError = false;
@@ -455,20 +462,18 @@ ElemFence VulkanExecuteCommandLists(ElemCommandQueue commandQueue, ElemCommandLi
     if (!hasError)
     {
         uint32_t signalCount = 1u;
+        auto signalPresentSemaphore = commandQueueData->SignalPresentSemaphore;
 
-        // TODO: Here we signal the present semaphore. We should do the same for the wait semaphore that we set during the acquire
-        // It is the same logic
-        // For both signalpresent and waitacquire, we need to have one per frame in flight
-        if (commandQueueData->SignalPresentSemaphore)
+        if (signalPresentSemaphore)
         {
+            SystemAssert(commandQueueData->PresentSemaphore != VK_NULL_HANDLE);
             signalCount = 2u;
-            commandQueueData->SignalPresentSemaphore = false;
         }
 
         uint64_t signalValues[] = { fenceValue, 0u };
 
         VkTimelineSemaphoreSubmitInfo timelineInfo = { VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO };
-        timelineInfo.waitSemaphoreValueCount = waitSemaphoreValues.Length;
+        timelineInfo.waitSemaphoreValueCount = waitSemaphores.Length;
         timelineInfo.pWaitSemaphoreValues = waitSemaphoreValues.Pointer;
         timelineInfo.signalSemaphoreValueCount = signalCount;
         timelineInfo.pSignalSemaphoreValues = signalValues;
@@ -486,6 +491,17 @@ ElemFence VulkanExecuteCommandLists(ElemCommandQueue commandQueue, ElemCommandLi
         submitInfo.pNext = &timelineInfo;
 
         AssertIfFailed(vkQueueSubmit(commandQueueData->DeviceObject, 1, &submitInfo, VK_NULL_HANDLE));
+
+        if (hasAcquireSemaphore)
+        {
+            commandQueueData->AcquireSemaphore = VK_NULL_HANDLE;
+        }
+
+        if (signalPresentSemaphore)
+        {
+            commandQueueData->SignalPresentSemaphore = false;
+            commandQueueData->PresentSemaphore = VK_NULL_HANDLE;
+        }
     }
 
     auto fence = ElemFence();
@@ -526,25 +542,15 @@ void VulkanWaitForFenceOnCpu(ElemFence fence)
     auto graphicsDeviceData = GetVulkanGraphicsDeviceData(commandQueueToWaitData->GraphicsDevice);
     SystemAssert(graphicsDeviceData);
 
-    if (fence.FenceValue > commandQueueToWaitData->LastCompletedFenceValue) 
-    {
-        uint64_t semaphoreValue;
-        vkGetSemaphoreCounterValue(graphicsDeviceData->Device, commandQueueToWaitData->Fence, &semaphoreValue);
-
-        commandQueueToWaitData->LastCompletedFenceValue = SystemMax(commandQueueToWaitData->LastCompletedFenceValue, semaphoreValue);
-    }
-
     if (fence.FenceValue > commandQueueToWaitData->LastCompletedFenceValue)
     {
-        // TODO: Activate it in a special debug mode
-        //SystemLogDebugMessage(ElemLogMessageCategory_Graphics, "Wait for fence on CPU...");
-
         VkSemaphoreWaitInfo waitInfo = { VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO };
         waitInfo.semaphoreCount = 1;
         waitInfo.pSemaphores = &commandQueueToWaitData->Fence;
         waitInfo.pValues = &fence.FenceValue;
 
         AssertIfFailed(vkWaitSemaphores(graphicsDeviceData->Device, &waitInfo, UINT64_MAX));
+        commandQueueToWaitData->LastCompletedFenceValue = fence.FenceValue;
     }
 }
 
@@ -644,4 +650,4 @@ void VulkanInsertGraphicsTimestamp(ElemCommandList commandList, ElemGraphicsTime
     commandListData->MaxResolveQueryIndex = SystemMax(timestampData->QueryHeapIndex, commandListData->MaxResolveQueryIndex);
 
     //vkCmdWriteTimestamp(commandListData->DeviceObject, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, graphicsDeviceData->QueryHeap.Storage->QueryHeap, timestampData->QueryHeapIndex);
-}
+}

src/Elemental/Common/Graphics/Vulkan/VulkanCommandList.h

@@ -23,6 +23,7 @@ struct VulkanCommandQueueData
     ElemGraphicsDevice GraphicsDevice;
     VkSemaphore Fence;
     uint64_t FenceValue;
+    VkSemaphore AcquireSemaphore;
     VkSemaphore PresentSemaphore;
     bool SignalPresentSemaphore;
     uint64_t LastCompletedFenceValue;

src/Elemental/Common/Graphics/Vulkan/VulkanGraphicsDevice.cpp

@@ -110,7 +110,8 @@ void InitVulkan()
             auto currentEnabledValidationFeaturesIndex = 0u;
 
             enabledValidationFeatures[currentEnabledValidationFeaturesIndex++] = VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT;
-            enabledValidationFeatures[currentEnabledValidationFeaturesIndex++] = VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT;
+
+            // TODO: Re-enable synchronization validation once submit-time validation no longer causes intermittent device loss.
 
             if (vulkanDebugGpuValidationEnabled)
             {

src/Elemental/Common/Graphics/Vulkan/VulkanResourceBarrier.cpp

@@ -102,8 +102,9 @@ void InsertVulkanResourceBarriersIfNeeded(ElemCommandList commandList, ElemGraph
     SystemAssert(commandListData);
 
     auto barriersInfo = GenerateBarrierCommands(stackMemoryArena, commandListData->ResourceBarrierPool, currentStage, VulkanDebugBarrierInfoEnabled);
+    auto needsRaytracingBuildBarrier = currentStage == ElemGraphicsResourceBarrierSyncType_BuildRaytracingAccelerationStructure;
 
-    if (barriersInfo.BufferBarriers.Length == 0 && barriersInfo.TextureBarriers.Length == 0)
+    if (barriersInfo.BufferBarriers.Length == 0 && barriersInfo.TextureBarriers.Length == 0 && !needsRaytracingBuildBarrier)
     {
         return;
     }
@@ -112,7 +113,7 @@ void InsertVulkanResourceBarriersIfNeeded(ElemCommandList commandList, ElemGraph
     dependencyInfo.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
 
     uint32_t vulkanBufferBarrierCount = 0;
-    bool hasAccelerationStructureBarrier = false;
+    bool hasAccelerationStructureBarrier = needsRaytracingBuildBarrier;
 
     for (uint32_t i = 0; i < barriersInfo.BufferBarriers.Length; i++)
     {
@@ -137,6 +138,23 @@ void InsertVulkanResourceBarriersIfNeeded(ElemCommandList commandList, ElemGraph
 
     VkMemoryBarrier2 accelerationStructureMemoryBarrier = { VK_STRUCTURE_TYPE_MEMORY_BARRIER_2 };
 
+    if (needsRaytracingBuildBarrier)
+    {
+        // HACK: Acceleration-structure builds implicitly read geometry/instance buffers and
+        // write acceleration-structure/scratch memory. Until the common barrier model records
+        // those command accesses explicitly, use one coarse phase barrier here so uploads and
+        // previous builds are visible to the next build. Replace this with the planned
+        // renderer-level/global synchronization model rather than growing per-BLAS tracking.
+        accelerationStructureMemoryBarrier.srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT |
+                                                          VK_PIPELINE_STAGE_2_ACCELERATION_STRUCTURE_BUILD_BIT_KHR;
+        accelerationStructureMemoryBarrier.dstStageMask = VK_PIPELINE_STAGE_2_ACCELERATION_STRUCTURE_BUILD_BIT_KHR;
+        accelerationStructureMemoryBarrier.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT |
+                                                           VK_ACCESS_2_ACCELERATION_STRUCTURE_WRITE_BIT_KHR;
+        accelerationStructureMemoryBarrier.dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT |
+                                                           VK_ACCESS_2_ACCELERATION_STRUCTURE_READ_BIT_KHR |
+                                                           VK_ACCESS_2_ACCELERATION_STRUCTURE_WRITE_BIT_KHR;
+    }
+
     if (hasAccelerationStructureBarrier)
     {
         dependencyInfo.pMemoryBarriers = &accelerationStructureMemoryBarrier;