From 97079318788a00211d337a88f0a4db381d070694 Mon Sep 17 00:00:00 2001 From: vladislavbelov Date: Mon, 5 Jun 2023 16:32:18 +0000 Subject: [PATCH] Fixes Vulkan hazards reported by validation layers. Tested By: Stan Differential Revision: https://code.wildfiregames.com/D5024 This was SVN commit r27665. --- binaries/data/config/default.cfg | 17 ++++++--- .../backend/vulkan/DeviceCommandContext.cpp | 37 ++++++++++++------- .../backend/vulkan/DeviceCommandContext.h | 2 + .../backend/vulkan/RingCommandContext.cpp | 2 +- .../backend/vulkan/SubmitScheduler.cpp | 15 ++++++++ .../renderer/backend/vulkan/SubmitScheduler.h | 4 ++ source/renderer/backend/vulkan/SwapChain.cpp | 2 +- source/renderer/backend/vulkan/Utilities.cpp | 2 +- source/renderer/backend/vulkan/Utilities.h | 2 +- 9 files changed, 59 insertions(+), 24 deletions(-) diff --git a/binaries/data/config/default.cfg b/binaries/data/config/default.cfg index 341fdadca2..c23048832a 100644 --- a/binaries/data/config/default.cfg +++ b/binaries/data/config/default.cfg @@ -126,16 +126,21 @@ cursorbackend = "sdl" rendererbackend = "gl" ; Enables additional debug information in renderer backend. -renderer.backend.debugcontext = "false" -renderer.backend.debugmessages = "false" -renderer.backend.debuglabels = "false" -renderer.backend.debugscopedlabels = "false" +renderer.backend.debugcontext = false +renderer.backend.debugmessages = false +renderer.backend.debuglabels = false +renderer.backend.debugscopedlabels = false -renderer.backend.gl.enableframebufferinvalidating = "false" +renderer.backend.gl.enableframebufferinvalidating = false -renderer.backend.vulkan.disabledescriptorindexing = "false" +renderer.backend.vulkan.disabledescriptorindexing = false renderer.backend.vulkan.deviceindexoverride = -1 +renderer.backend.vulkan.debugbarrierafterframebufferpass = false +renderer.backend.vulkan.debugwaitidlebeforeacquire = false +renderer.backend.vulkan.debugwaitidlebeforepresent = false +renderer.backend.vulkan.debugwaitidleafterpresent = false + ; Should not be edited. It's used only for preventing of running fixed pipeline. renderpath = default diff --git a/source/renderer/backend/vulkan/DeviceCommandContext.cpp b/source/renderer/backend/vulkan/DeviceCommandContext.cpp index de718c274a..0ee3fa5d6a 100644 --- a/source/renderer/backend/vulkan/DeviceCommandContext.cpp +++ b/source/renderer/backend/vulkan/DeviceCommandContext.cpp @@ -22,6 +22,7 @@ #include "lib/bits.h" #include "maths/MathUtil.h" #include "ps/CLogger.h" +#include "ps/ConfigDB.h" #include "ps/containers/Span.h" #include "ps/containers/StaticVector.h" #include "renderer/backend/vulkan/Buffer.h" @@ -276,6 +277,16 @@ void CDeviceCommandContext::CUploadRing::ExecuteUploads( if (m_BlockOffset == 0) return; + const VkPipelineStageFlags stageMask = + m_Type == IBuffer::Type::UNIFORM + ? VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT + : VK_PIPELINE_STAGE_VERTEX_INPUT_BIT; + + Utilities::SubmitBufferMemoryBarrier( + commandBuffer, m_Buffer.get(), 0, VK_WHOLE_SIZE, + VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + stageMask, VK_PIPELINE_STAGE_TRANSFER_BIT); + VkBufferCopy region{}; region.srcOffset = m_BlockIndex * m_Capacity; region.dstOffset = 0; @@ -286,19 +297,10 @@ void CDeviceCommandContext::CUploadRing::ExecuteUploads( m_StagingBuffer->GetVkBuffer(), m_Buffer->GetVkBuffer(), 1, ®ion); - VkMemoryBarrier memoryBarrier{}; - memoryBarrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; - memoryBarrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; - memoryBarrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; - - const VkPipelineStageFlags dstStageMask = - m_Type == IBuffer::Type::UNIFORM - ? VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT - : VK_PIPELINE_STAGE_VERTEX_INPUT_BIT; - vkCmdPipelineBarrier( - commandBuffer, - VK_PIPELINE_STAGE_TRANSFER_BIT, dstStageMask, 0, - 1, &memoryBarrier, 0, nullptr, 0, nullptr); + Utilities::SubmitBufferMemoryBarrier( + commandBuffer, m_Buffer.get(), 0, VK_WHOLE_SIZE, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, stageMask); m_BlockIndex = (m_BlockIndex + 1) % NUMBER_OF_FRAMES_IN_FLIGHT; m_BlockOffset = 0; @@ -365,6 +367,10 @@ std::unique_ptr CDeviceCommandContext::Create(CDevice* de vkUpdateDescriptorSets( device->GetVkDevice(), 1, &writeDescriptorSet, 0, nullptr); + CFG_GET_VAL( + "renderer.backend.vulkan.debugbarrierafterframebufferpass", + deviceCommandContext->m_DebugBarrierAfterFramebufferPass); + return deviceCommandContext; } @@ -667,6 +673,9 @@ void CDeviceCommandContext::EndFramebufferPass() if (m_ShaderProgram) m_ShaderProgram->Unbind(); m_ShaderProgram = nullptr; + + if (m_DebugBarrierAfterFramebufferPass) + Utilities::SubmitDebugSyncMemoryBarrier(m_CommandContext->GetCommandBuffer()); } void CDeviceCommandContext::ReadbackFramebufferSync( @@ -894,7 +903,7 @@ void CDeviceCommandContext::SetTexture(const int32_t bindingSlot, ITexture* text if (!m_Device->GetDescriptorManager().UseDescriptorIndexing()) { - // We can't bind textures which are used as color attachments. + // We can't bind textures which are used as attachments. const auto& colorAttachments = m_Framebuffer->GetColorAttachments(); ENSURE(std::find( colorAttachments.begin(), colorAttachments.end(), textureToBind) == colorAttachments.end()); diff --git a/source/renderer/backend/vulkan/DeviceCommandContext.h b/source/renderer/backend/vulkan/DeviceCommandContext.h index 6e0a93a1b5..746ba415b3 100644 --- a/source/renderer/backend/vulkan/DeviceCommandContext.h +++ b/source/renderer/backend/vulkan/DeviceCommandContext.h @@ -186,6 +186,8 @@ private: void* data = nullptr; }; PS::StaticVector m_QueuedReadbacks; + + bool m_DebugBarrierAfterFramebufferPass = false; }; } // namespace Vulkan diff --git a/source/renderer/backend/vulkan/RingCommandContext.cpp b/source/renderer/backend/vulkan/RingCommandContext.cpp index 1952866c7d..527793c81b 100644 --- a/source/renderer/backend/vulkan/RingCommandContext.cpp +++ b/source/renderer/backend/vulkan/RingCommandContext.cpp @@ -313,7 +313,7 @@ void CRingCommandContext::Begin() VkCommandBufferBeginInfo beginInfo{}; beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; - beginInfo.flags = 0; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; beginInfo.pInheritanceInfo = nullptr; ENSURE_VK_SUCCESS(vkBeginCommandBuffer(item.commandBuffer, &beginInfo)); } diff --git a/source/renderer/backend/vulkan/SubmitScheduler.cpp b/source/renderer/backend/vulkan/SubmitScheduler.cpp index c7ca1d4ea0..cdaf5de8e7 100644 --- a/source/renderer/backend/vulkan/SubmitScheduler.cpp +++ b/source/renderer/backend/vulkan/SubmitScheduler.cpp @@ -19,6 +19,7 @@ #include "SubmitScheduler.h" +#include "ps/ConfigDB.h" #include "renderer/backend/vulkan/Device.h" #include "renderer/backend/vulkan/RingCommandContext.h" #include "renderer/backend/vulkan/SwapChain.h" @@ -66,6 +67,10 @@ CSubmitScheduler::CSubmitScheduler( device, NUMBER_OF_FRAMES_IN_FLIGHT, queueFamilyIndex, *this); m_PresentCommandContext = std::make_unique( device, NUMBER_OF_FRAMES_IN_FLIGHT, queueFamilyIndex, *this); + + CFG_GET_VAL("renderer.backend.vulkan.debugwaitidlebeforeacquire", m_DebugWaitIdleBeforeAcquire); + CFG_GET_VAL("renderer.backend.vulkan.debugwaitidlebeforepresent", m_DebugWaitIdleBeforePresent); + CFG_GET_VAL("renderer.backend.vulkan.debugwaitidleafterpresent", m_DebugWaitIdleAfterPresent); } CSubmitScheduler::~CSubmitScheduler() @@ -88,6 +93,9 @@ CSubmitScheduler::~CSubmitScheduler() bool CSubmitScheduler::AcquireNextImage(CSwapChain& swapChain) { + if (m_DebugWaitIdleBeforeAcquire) + vkDeviceWaitIdle(m_Device->GetVkDevice()); + FrameObject& frameObject = m_FrameObjects[m_FrameID % m_FrameObjects.size()]; if (!swapChain.AcquireNextImage(frameObject.acquireImageSemaphore)) return false; @@ -106,7 +114,14 @@ void CSubmitScheduler::Present(CSwapChain& swapChain) m_NextSubmitSignalSemaphore = frameObject.submitDone; m_PresentCommandContext->Flush(); Flush(); + + if (m_DebugWaitIdleBeforePresent) + vkDeviceWaitIdle(m_Device->GetVkDevice()); + swapChain.Present(frameObject.submitDone, m_Queue); + + if (m_DebugWaitIdleAfterPresent) + vkDeviceWaitIdle(m_Device->GetVkDevice()); } CSubmitScheduler::SubmitHandle CSubmitScheduler::Submit(VkCommandBuffer commandBuffer) diff --git a/source/renderer/backend/vulkan/SubmitScheduler.h b/source/renderer/backend/vulkan/SubmitScheduler.h index bc8f1ea455..325fdea15f 100644 --- a/source/renderer/backend/vulkan/SubmitScheduler.h +++ b/source/renderer/backend/vulkan/SubmitScheduler.h @@ -106,6 +106,10 @@ private: std::unique_ptr m_AcquireCommandContext; std::unique_ptr m_PresentCommandContext; + + bool m_DebugWaitIdleBeforeAcquire = false; + bool m_DebugWaitIdleBeforePresent = false; + bool m_DebugWaitIdleAfterPresent = false; }; } // namespace Vulkan diff --git a/source/renderer/backend/vulkan/SwapChain.cpp b/source/renderer/backend/vulkan/SwapChain.cpp index 4386068975..bd8a8c55db 100644 --- a/source/renderer/backend/vulkan/SwapChain.cpp +++ b/source/renderer/backend/vulkan/SwapChain.cpp @@ -283,7 +283,7 @@ void CSwapChain::SubmitCommandsAfterAcquireNextImage( 0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, firstAcquirement ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, - firstAcquirement ? VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT : VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); if (!m_DepthTexture->IsInitialized()) { Utilities::SubmitImageMemoryBarrier( diff --git a/source/renderer/backend/vulkan/Utilities.cpp b/source/renderer/backend/vulkan/Utilities.cpp index d0a4f37788..358c58c58d 100644 --- a/source/renderer/backend/vulkan/Utilities.cpp +++ b/source/renderer/backend/vulkan/Utilities.cpp @@ -96,7 +96,7 @@ void SubmitImageMemoryBarrier( void SubmitBufferMemoryBarrier( VkCommandBuffer commandBuffer, CBuffer* buffer, - const uint32_t offset, const uint32_t size, + const VkDeviceSize offset, const VkDeviceSize size, const VkAccessFlags srcAccessMask, const VkAccessFlags dstAccessMask, const VkPipelineStageFlags srcStageMask, const VkPipelineStageFlags dstStageMask) { diff --git a/source/renderer/backend/vulkan/Utilities.h b/source/renderer/backend/vulkan/Utilities.h index 26899f027a..3de1746a88 100644 --- a/source/renderer/backend/vulkan/Utilities.h +++ b/source/renderer/backend/vulkan/Utilities.h @@ -65,7 +65,7 @@ void SubmitImageMemoryBarrier( void SubmitBufferMemoryBarrier( VkCommandBuffer commandBuffer, CBuffer* buffer, - const uint32_t offset, const uint32_t size, + const VkDeviceSize offset, const VkDeviceSize size, const VkAccessFlags srcAccessMask, const VkAccessFlags dstAccessMask, const VkPipelineStageFlags srcStageMask, const VkPipelineStageFlags dstStageMask);