looks like command buffers need to be externally synchronized

src/Engine/Asset/AssetRegistry.cpp

@@ -5,6 +5,7 @@
 #include "MaterialLoader.h"
 #include "MeshLoader.h"
 #include "Material/MaterialAsset.h"
+#include "Graphics/Mesh.h"
 #include "Graphics/Graphics.h"
 #include "Window/WindowManager.h"
 #include "MeshAsset.h"

src/Engine/Containers/Array.h

@@ -179,7 +179,7 @@ namespace Seele
                     }
                     allocator = other.allocator;
                 }
-                if(other.arraySize > allocated)
+                if(_data == nullptr || other.arraySize > allocated)
                 {
                     if(_data != nullptr)
                     {

src/Engine/Containers/Map.h

@@ -502,7 +502,7 @@ private:
         if (!isValid(r))
         {
             nodeContainer.emplace(std::forward<KeyType>(key));
-            return 0;
+            return nodeContainer.size() - 1;
         }
         r = splay(r, key);
         Node* node = getNode(r);

src/Engine/Graphics/CMakeLists.txt

@@ -10,6 +10,7 @@ target_sources(SeeleEngine
         Mesh.h
         Mesh.cpp
         MeshBatch.h
+        MeshBatch.cpp
         RenderMaterial.h
         RenderMaterial.cpp
         ShaderCompiler.h

src/Engine/Graphics/GraphicsResources.h

@@ -3,7 +3,6 @@
 #include "Containers/Array.h"
 #include "Containers/List.h"
 #include "GraphicsInitializer.h"
-#include "MeshBatch.h"
 #include <boost/crc.hpp>
 #include <functional>
 
@@ -17,6 +16,8 @@ namespace Seele
 struct VertexInputStream;
 struct VertexStreamComponent;
 class VertexInputType;
+struct MeshBatchElement;
+DECLARE_REF(MaterialAsset)
 namespace Gfx
 {
 DECLARE_REF(Graphics)

src/Engine/Graphics/MeshBatch.cpp

@@ -0,0 +1,28 @@
+#include "MeshBatch.h"
+#include "GraphicsResources.h"
+#include "VertexShaderInput.h"
+#include "Material/MaterialAsset.h"
+
+using namespace Seele;
+
+MeshBatchElement::MeshBatchElement()
+    : uniformBuffer(nullptr)
+    , indexBuffer(nullptr)
+    , firstIndex(0)
+    , numPrimitives(0)
+    , numInstances(1)
+    , baseVertexIndex(0)
+    , minVertexIndex(0)
+    , maxVertexIndex(0)
+    , indirectArgsBuffer(nullptr)
+{}
+MeshBatch::MeshBatch()
+    : useReverseCulling(false)
+    , isBackfaceCullingDisabled(false)
+    , isCastingShadow(true)
+    , useWireframe(false)
+    , topology(Gfx::SE_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
+    , vertexInput(nullptr)
+    , material(nullptr)
+{
+}

src/Engine/Graphics/MeshBatch.h

@@ -1,9 +1,10 @@
 #pragma once
+#include "GraphicsEnums.h"
 
 namespace Seele
 {
-DECLARE_REF(MaterialAsset)
 DECLARE_REF(VertexShaderInput)
+DECLARE_REF(MaterialAsset)
 DECLARE_NAME_REF(Gfx, VertexBuffer)
 DECLARE_NAME_REF(Gfx, IndexBuffer)
 DECLARE_NAME_REF(Gfx, UniformBuffer)
@@ -29,17 +30,7 @@ public:
 
     uint8 isInstanced : 1;
     Gfx::PVertexBuffer indirectArgsBuffer;
-    MeshBatchElement()
+    MeshBatchElement();
-        : uniformBuffer(nullptr)
-        , indexBuffer(nullptr)
-        , firstIndex(0)
-        , numPrimitives(0)
-        , numInstances(1)
-        , baseVertexIndex(0)
-        , minVertexIndex(0)
-        , maxVertexIndex(0)
-        , indirectArgsBuffer(nullptr)
-    {}
 };
 struct MeshBatch
 {
@@ -76,16 +67,7 @@ struct MeshBatch
         return count;
     }
 
-    MeshBatch()
+    MeshBatch();
-        : useReverseCulling(false)
-        , isBackfaceCullingDisabled(false)
-        , isCastingShadow(true)
-        , useWireframe(false)
-        , topology(Gfx::SE_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
-        , vertexInput(nullptr)
-        , material(nullptr)
-    {
-    }
 };
 DECLARE_REF(PrimitiveComponent)
 struct StaticMeshBatch : public MeshBatch

src/Engine/Graphics/ShaderCompiler.cpp

@@ -1,6 +1,7 @@
 #include "ShaderCompiler.h"
 #include "Material/MaterialAsset.h"
 #include "VertexShaderInput.h"
+#include "Graphics.h"
 
 using namespace Seele;
 using namespace Seele::Gfx;

src/Engine/Graphics/Vulkan/VulkanAllocator.cpp

@@ -131,11 +131,19 @@ void Allocation::markFree(SubAllocation *allocation)
     VkDeviceSize lowerBound = allocation->allocatedOffset;
     VkDeviceSize upperBound = allocation->allocatedOffset + allocation->allocatedSize;
     PSubAllocation allocHandle;
+    PSubAllocation freeRangeToDelete;
+
     std::unique_lock lck(lock);
     //Join lower bound
     for (auto freeRange : freeRanges)
     {
         PSubAllocation freeAlloc = freeRange.value;
+        if (freeAlloc->allocatedOffset <= lowerBound
+        && freeAlloc->allocatedOffset + freeAlloc->allocatedSize >= upperBound)
+        {
+            // allocation is already in a free region
+            return;
+        }
         if (freeAlloc->allocatedOffset + freeAlloc->allocatedSize == lowerBound)
         {
             //extend freeAlloc by the allocatedSize
@@ -148,6 +156,7 @@ void Allocation::markFree(SubAllocation *allocation)
     auto foundAlloc = freeRanges.find(upperBound);
     if (foundAlloc != freeRanges.end())
     {
+        freeRangeToDelete = foundAlloc->value;
         // There is a free allocation ending where the new free one ends
         if (allocHandle != nullptr)
         {
@@ -159,22 +168,23 @@ void Allocation::markFree(SubAllocation *allocation)
         {
             // set foundAlloc back by size amount
             allocHandle = foundAlloc->value;
-			// remove from offset map since key changes
-			freeRanges.erase(foundAlloc->key);
             allocHandle->allocatedOffset -= allocation->allocatedSize;
             allocHandle->alignedOffset -= allocation->allocatedSize;
             // place back at correct offset
             freeRanges[allocHandle->allocatedOffset] = allocHandle;
+            // remove from offset map since key changes
+            freeRanges.erase(foundAlloc->key);
         }
     }
+    
     if (allocHandle == nullptr)
     {
         allocHandle = new SubAllocation(this, allocation->alignedOffset, allocation->size, allocation->alignedOffset, allocation->allocatedSize);
         freeRanges[allocation->allocatedOffset] = allocHandle;
     }
     activeAllocations.erase(allocation->allocatedOffset);
+    lock.unlock();
     bytesUsed -= allocation->allocatedSize;
-
     // TODO: delete allocation when bytesUsed == 0
 }
 

src/Engine/Graphics/Vulkan/VulkanBuffer.cpp

@@ -3,6 +3,7 @@
 #include "VulkanGraphics.h"
 #include "VulkanCommandBuffer.h"
 #include "VulkanAllocator.h"
+#include "ThreadPool.h"
 
 using namespace Seele;
 using namespace Seele::Vulkan;
@@ -62,14 +63,17 @@ ShaderBuffer::ShaderBuffer(PGraphics graphics, uint32 size, VkBufferUsageFlags u
 
 ShaderBuffer::~ShaderBuffer()
 {
-	auto cmdBuffer = graphics->getQueueCommands(owner)->getCommands();
+	PCmdBuffer cmdBuffer = graphics->getQueueCommands(owner)->getCommands();
-	auto &deletionQueue = graphics->getDeletionQueue();
 	VkDevice device = graphics->getDevice();
-	VkBuffer buf[Gfx::numFramesBuffered];
+	auto deletionLambda = [](PCmdBuffer cmdBuffer, VkDevice device, VkBuffer buffer) -> Job 
+	{
+		co_await cmdBuffer->asyncWait();
+		vkDestroyBuffer(device, buffer, nullptr);
+		co_return;
+	};
 	for (uint32 i = 0; i < numBuffers; ++i)
 	{
-		buf[i] = buffers[i].buffer;
+		deletionLambda(cmdBuffer, device, buffers[i].buffer);
-		deletionQueue.addPendingDelete(cmdBuffer, [device, buf, i]() { vkDestroyBuffer(device, buf[i], nullptr); });
 		buffers[i].allocation = nullptr;
 	}
 	graphics = nullptr;

src/Engine/Graphics/Vulkan/VulkanCommandBuffer.cpp

@@ -25,6 +25,7 @@ CmdBuffer::CmdBuffer(PGraphics graphics, VkCommandPool cmdPool, PCommandBufferMa
         init::CommandBufferAllocateInfo(cmdPool,
                                         VK_COMMAND_BUFFER_LEVEL_PRIMARY,
                                         1);
+    std::unique_lock lock(handleLock);
     VK_CHECK(vkAllocateCommandBuffers(graphics->getDevice(), &allocInfo, &handle))
 
     fence = new Fence(graphics);
@@ -33,6 +34,7 @@ CmdBuffer::CmdBuffer(PGraphics graphics, VkCommandPool cmdPool, PCommandBufferMa
 
 CmdBuffer::~CmdBuffer()
 {
+    std::unique_lock lock(handleLock);
     vkFreeCommandBuffers(graphics->getDevice(), owner, 1, &handle);
     renderPass = nullptr;
     framebuffer = nullptr;
@@ -44,18 +46,21 @@ void CmdBuffer::begin()
     VkCommandBufferBeginInfo beginInfo =
         init::CommandBufferBeginInfo();
     beginInfo.pInheritanceInfo = nullptr;
+    std::unique_lock lock(handleLock);
     VK_CHECK(vkBeginCommandBuffer(handle, &beginInfo));
     state = State::InsideBegin;
 }
 
 void CmdBuffer::end()
 {
+    std::unique_lock lock(handleLock);
     VK_CHECK(vkEndCommandBuffer(handle));
     state = State::Ended;
 }
 
 void CmdBuffer::beginRenderPass(PRenderPass newRenderPass, PFramebuffer newFramebuffer)
 {
+    std::unique_lock lock(handleLock);
     renderPass = newRenderPass;
     framebuffer = newFramebuffer;
 
@@ -72,6 +77,7 @@ void CmdBuffer::beginRenderPass(PRenderPass newRenderPass, PFramebuffer newFrame
 
 void CmdBuffer::endRenderPass()
 {
+    std::unique_lock lock(handleLock);
     vkCmdEndRenderPass(handle);
     state = State::InsideBegin;
 }
@@ -79,6 +85,7 @@ void CmdBuffer::endRenderPass()
 void CmdBuffer::executeCommands(const Array<Gfx::PRenderCommand>& commands)
 {
     assert(state == State::RenderPassActive);
+    std::unique_lock lock(handleLock);
     Array<VkCommandBuffer> cmdBuffers(commands.size());
     for (uint32 i = 0; i < commands.size(); ++i)
     {
@@ -97,6 +104,7 @@ void CmdBuffer::executeCommands(const Array<Gfx::PRenderCommand>& commands)
 
 void CmdBuffer::executeCommands(const Array<Gfx::PComputeCommand>& commands) 
 {
+    std::unique_lock lock(handleLock);
     Array<VkCommandBuffer> cmdBuffers(commands.size());
     for (uint32 i = 0; i < commands.size(); ++i)
     {
@@ -115,12 +123,14 @@ void CmdBuffer::executeCommands(const Array<Gfx::PComputeCommand>& commands)
 
 void CmdBuffer::addWaitSemaphore(VkPipelineStageFlags flags, PSemaphore semaphore)
 {
+    std::unique_lock lock(handleLock);
     waitSemaphores.add(semaphore);
     waitFlags.add(flags);
 }
 
 void CmdBuffer::refreshFence()
 {
+    std::unique_lock lock(handleLock);
     if (state == State::Submitted)
     {
         if (fence->isSignaled())
@@ -153,10 +163,16 @@ void CmdBuffer::refreshFence()
 
 void CmdBuffer::waitForCommand(uint32 timeout)
 {
+    std::unique_lock lock(handleLock);
     fence->wait(timeout);
     refreshFence();
 }
 
+Event CmdBuffer::asyncWait() const
+{
+    return fence->asyncWait();
+}
+
 
 PFence CmdBuffer::getFence()
 {
@@ -187,6 +203,7 @@ RenderCommand::~RenderCommand()
 
 void RenderCommand::begin(PCmdBuffer parent)
 {
+    threadId = std::this_thread::get_id();
     ready = false;
     VkCommandBufferBeginInfo beginInfo =
         init::CommandBufferBeginInfo();
@@ -202,11 +219,13 @@ void RenderCommand::begin(PCmdBuffer parent)
 
 void RenderCommand::end() 
 {
+    assert(threadId == std::this_thread::get_id());
     VK_CHECK(vkEndCommandBuffer(handle));
 }
 
 void RenderCommand::reset() 
 {
+    assert(threadId == std::this_thread::get_id());
     vkResetCommandBuffer(handle, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
     boundDescriptors.clear();
     ready = true;
@@ -219,6 +238,7 @@ bool RenderCommand::isReady()
 
 void RenderCommand::setViewport(Gfx::PViewport viewport) 
 {
+    assert(threadId == std::this_thread::get_id());
     VkViewport vp = viewport.cast<Viewport>()->getHandle();
     VkRect2D scissors = init::Rect2D(viewport->getSizeX(), viewport->getSizeY(), viewport->getOffsetX(), viewport->getOffsetY());
     vkCmdSetViewport(handle, 0, 1, &vp);
@@ -227,11 +247,13 @@ void RenderCommand::setViewport(Gfx::PViewport viewport)
 
 void RenderCommand::bindPipeline(Gfx::PGraphicsPipeline gfxPipeline)
 {
+    assert(threadId == std::this_thread::get_id());
     pipeline = gfxPipeline.cast<GraphicsPipeline>();
     pipeline->bind(handle);
 }
 void RenderCommand::bindDescriptor(Gfx::PDescriptorSet descriptorSet)
 {
+    assert(threadId == std::this_thread::get_id());
     auto descriptor = descriptorSet.cast<DescriptorSet>();
     boundDescriptors.add(descriptor.getHandle());
     descriptor->bind();
@@ -241,6 +263,7 @@ void RenderCommand::bindDescriptor(Gfx::PDescriptorSet descriptorSet)
 }
 void RenderCommand::bindDescriptor(const Array<Gfx::PDescriptorSet>& descriptorSets)
 {
+    assert(threadId == std::this_thread::get_id());
     VkDescriptorSet* sets = new VkDescriptorSet[descriptorSets.size()];
     for(uint32 i = 0; i < descriptorSets.size(); ++i)
     {
@@ -255,6 +278,7 @@ void RenderCommand::bindDescriptor(const Array<Gfx::PDescriptorSet>& descriptorS
 }
 void RenderCommand::bindVertexBuffer(const Array<VertexInputStream>& streams)
 {
+    assert(threadId == std::this_thread::get_id());
     Array<VkBuffer> buffers(streams.size());
     Array<VkDeviceSize> offsets(streams.size());
     for(uint32 i = 0; i < streams.size(); ++i)
@@ -267,16 +291,19 @@ void RenderCommand::bindVertexBuffer(const Array<VertexInputStream>& streams)
 }
 void RenderCommand::bindIndexBuffer(Gfx::PIndexBuffer indexBuffer)
 {
+    assert(threadId == std::this_thread::get_id());
     PIndexBuffer buf = indexBuffer.cast<IndexBuffer>();
     vkCmdBindIndexBuffer(handle, buf->getHandle(), 0, cast(buf->getIndexType()));
 }
 void RenderCommand::draw(const MeshBatchElement& data) 
 {
+    assert(threadId == std::this_thread::get_id());
     vkCmdDrawIndexed(handle, data.indexBuffer->getNumIndices(), data.numInstances, data.minVertexIndex, data.baseVertexIndex, 0);
 }
 
 void RenderCommand::draw(uint32 vertexCount, uint32 instanceCount, int32 firstVertex, uint32 firstInstance) 
 {
+    assert(threadId == std::this_thread::get_id());
     vkCmdDraw(handle, vertexCount, instanceCount, firstVertex, firstInstance);
 }
 
@@ -299,6 +326,7 @@ ComputeCommand::~ComputeCommand()
 
 void ComputeCommand::begin(PCmdBuffer) 
 {
+    threadId = std::this_thread::get_id();
     ready = false;
     VkCommandBufferBeginInfo beginInfo =
         init::CommandBufferBeginInfo();
@@ -313,11 +341,13 @@ void ComputeCommand::begin(PCmdBuffer)
 
 void ComputeCommand::end() 
 {
+    assert(threadId == std::this_thread::get_id());
     VK_CHECK(vkEndCommandBuffer(handle));
 }
 
 void ComputeCommand::reset() 
 {
+    assert(threadId == std::this_thread::get_id());
     vkResetCommandBuffer(handle, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
     boundDescriptors.clear();
     ready = true;
@@ -329,12 +359,14 @@ bool ComputeCommand::isReady()
 
 void ComputeCommand::bindPipeline(Gfx::PComputePipeline computePipeline) 
 {
+    assert(threadId == std::this_thread::get_id());
     pipeline = computePipeline.cast<ComputePipeline>();
     pipeline->bind(handle);
 }
 
 void ComputeCommand::bindDescriptor(Gfx::PDescriptorSet descriptorSet) 
 {
+    assert(threadId == std::this_thread::get_id());
     auto descriptor = descriptorSet.cast<DescriptorSet>();
     boundDescriptors.add(descriptor.getHandle());
     descriptor->bind();
@@ -345,6 +377,7 @@ void ComputeCommand::bindDescriptor(Gfx::PDescriptorSet descriptorSet)
 
 void ComputeCommand::bindDescriptor(const Array<Gfx::PDescriptorSet>& descriptorSets) 
 {
+    assert(threadId == std::this_thread::get_id());
     VkDescriptorSet* sets = new VkDescriptorSet[descriptorSets.size()];
     for(uint32 i = 0; i < descriptorSets.size(); ++i)
     {
@@ -360,6 +393,7 @@ void ComputeCommand::bindDescriptor(const Array<Gfx::PDescriptorSet>& descriptor
 
 void ComputeCommand::dispatch(uint32 threadX, uint32 threadY, uint32 threadZ) 
 {
+    assert(threadId == std::this_thread::get_id());
     vkCmdDispatch(handle, threadX, threadY, threadZ);
 }
 

src/Engine/Graphics/Vulkan/VulkanCommandBuffer.h

@@ -32,6 +32,7 @@ public:
 	void addWaitSemaphore(VkPipelineStageFlags stages, PSemaphore waitSemaphore);
 	void refreshFence();
 	void waitForCommand(uint32 timeToWait = 1000000u);
+	Event asyncWait() const;
 	PFence getFence();
 	PCommandBufferManager getManager();
 	enum State
@@ -53,6 +54,7 @@ private:
 	State state;
 	VkViewport currentViewport;
 	VkRect2D currentScissor;
+	std::mutex handleLock;
 	VkCommandBuffer handle;
 	VkCommandPool owner;
 	Array<PSemaphore> waitSemaphores;
@@ -96,6 +98,7 @@ private:
 	VkViewport currentViewport;
 	VkRect2D currentScissor;
 	PGraphics graphics;
+	std::thread::id threadId;
 	VkCommandBuffer handle;
 	VkCommandPool owner;
 	friend class CmdBuffer;
@@ -126,6 +129,7 @@ private:
 	VkViewport currentViewport;
 	VkRect2D currentScissor;
 	PGraphics graphics;
+	std::thread::id threadId;
 	VkCommandBuffer handle;
 	VkCommandPool owner;
 	friend class CmdBuffer;

src/Engine/Graphics/Vulkan/VulkanFramebuffer.cpp

@@ -8,7 +8,9 @@ using namespace Seele;
 using namespace Seele::Vulkan;
 
 Framebuffer::Framebuffer(PGraphics graphics, PRenderPass renderPass, Gfx::PRenderTargetLayout renderTargetLayout)
-    : graphics(graphics), layout(renderTargetLayout), renderPass(renderPass)
+    : graphics(graphics)
+    , layout(renderTargetLayout)
+    , renderPass(renderPass)
 {
     FramebufferDescription description;
     std::memset(&description, 0, sizeof(FramebufferDescription));

src/Engine/Graphics/Vulkan/VulkanGraphics.cpp

@@ -52,6 +52,7 @@ Gfx::PWindow Graphics::createWindow(const WindowCreateInfo &createInfo)
 Gfx::PViewport Graphics::createViewport(Gfx::PWindow owner, const ViewportCreateInfo &viewportInfo)
 {
     PViewport result = new Viewport(this, owner, viewportInfo);
+    std::unique_lock lock(viewportLock);
     viewports.add(result);
     return result;
 }
@@ -65,6 +66,8 @@ void Graphics::beginRenderPass(Gfx::PRenderPass renderPass)
     PRenderPass rp = renderPass.cast<RenderPass>();
     uint32 framebufferHash = rp->getFramebufferHash();
     PFramebuffer framebuffer;
+    {
+        std::unique_lock lock(allocatedFrameBufferLock);
         auto found = allocatedFramebuffers.find(framebufferHash);
         if (found == allocatedFramebuffers.end())
         {
@@ -75,6 +78,7 @@ void Graphics::beginRenderPass(Gfx::PRenderPass renderPass)
         {
             framebuffer = found->value;
         }
+    }
     getGraphicsCommands()->getCommands()->beginRenderPass(rp, framebuffer);
 }
 
@@ -275,21 +279,6 @@ void Graphics::copyTexture(Gfx::PTexture srcTexture, Gfx::PTexture dstTexture)
     }
 }
 
-VkInstance Graphics::getInstance() const
-{
-	return instance;
-}
-
-VkDevice Graphics::getDevice() const
-{
-	return handle;
-}
-
-VkPhysicalDevice Graphics::getPhysicalDevice() const
-{
-	return physicalDevice;
-}
-
 PCommandBufferManager Graphics::getQueueCommands(Gfx::QueueType queueType)
 {
     switch (queueType)
@@ -360,7 +349,6 @@ PStagingManager Graphics::getStagingManager()
     return stagingManager;
 }
 
-
 Array<const char *> Graphics::getRequiredExtensions()
 {
     Array<const char *> extensions;
@@ -566,6 +554,7 @@ void Graphics::createDevice(GraphicsInitializer initializer)
     deviceInfo.ppEnabledLayerNames = initializer.layers.data();
 
     VK_CHECK(vkCreateDevice(physicalDevice, &deviceInfo, nullptr, &handle));
+    std::cout << "Vulkan handle: " << handle << std::endl;
 
     graphicsQueue = new Queue(this, Gfx::QueueType::GRAPHICS, graphicsQueueInfo.familyIndex, 0);
     if (Gfx::useAsyncCompute && asyncComputeInfo.familyIndex != -1)

src/Engine/Graphics/Vulkan/VulkanGraphics.h

@@ -19,9 +19,9 @@ class Graphics : public Gfx::Graphics
 public:
     Graphics();
     virtual ~Graphics();
-	VkInstance getInstance() const;
+    constexpr VkInstance getInstance() const { return instance; };
-	VkDevice getDevice() const;
+    constexpr VkDevice getDevice() const { return handle; };
-	VkPhysicalDevice getPhysicalDevice() const;
+    constexpr VkPhysicalDevice getPhysicalDevice() const { return physicalDevice; };
 
     PCommandBufferManager getQueueCommands(Gfx::QueueType queueType);
     PCommandBufferManager getGraphicsCommands();
@@ -29,11 +29,6 @@ public:
     PCommandBufferManager getTransferCommands();
     PCommandBufferManager getDedicatedTransferCommands();
 
-	QueueOwnedResourceDeletion &getDeletionQueue()
-	{
-		return deletionQueue;
-	}
-
     PAllocator getAllocator();
     PStagingManager getStagingManager();
 
@@ -86,7 +81,6 @@ protected:
     PQueue computeQueue;
     PQueue transferQueue;
     PQueue dedicatedTransferQueue;
-	QueueOwnedResourceDeletion deletionQueue;
     PPipelineCache pipelineCache;
     Map<std::thread::id, PCommandBufferManager> graphicsCommands;
     Map<std::thread::id, PCommandBufferManager> computeCommands;
@@ -95,7 +89,9 @@ protected:
     VkPhysicalDeviceProperties props;
     VkPhysicalDeviceFeatures features;
     VkDebugReportCallbackEXT callback;
+    std::mutex viewportLock;
     Array<PViewport> viewports;
+    std::mutex allocatedFrameBufferLock;
     Map<uint32, PFramebuffer> allocatedFramebuffers;
     PAllocator allocator;
     PStagingManager stagingManager;

src/Engine/Graphics/Vulkan/VulkanGraphicsResources.cpp

@@ -7,48 +7,6 @@
 using namespace Seele;
 using namespace Seele::Vulkan;
 
-List<QueueOwnedResourceDeletion::PendingItem> QueueOwnedResourceDeletion::deletionQueue;
-volatile bool QueueOwnedResourceDeletion::running;
-std::mutex QueueOwnedResourceDeletion::mutex;
-std::condition_variable QueueOwnedResourceDeletion::cv;
-
-QueueOwnedResourceDeletion::QueueOwnedResourceDeletion()
-{
-    worker = std::thread(&run);
-    running = true;
-}
-
-QueueOwnedResourceDeletion::~QueueOwnedResourceDeletion()
-{
-    worker.join();
-}
-
-void QueueOwnedResourceDeletion::addPendingDelete(PCmdBuffer cmdbuffer, std::function<void()> func)
-{
-    PendingItem item;
-    item.cmdBuffer = cmdbuffer;
-    item.func = func;
-    deletionQueue.add(item);
-    std::unique_lock<std::mutex> lock(mutex);
-    cv.notify_all();
-}
-
-void QueueOwnedResourceDeletion::run()
-{
-    while (running || !deletionQueue.empty())
-    {
-        std::unique_lock<std::mutex> lock(mutex);
-        cv.wait(lock);
-        auto entry = deletionQueue.begin();
-        PCmdBuffer cmdBuffer = entry->cmdBuffer;
-        //cmdBuffer->getManager()->waitForCommands(cmdBuffer);
-        //cmdBuffer->begin();
-
-        //entry->func();
-        deletionQueue.remove(entry);
-    }
-}
-
 Semaphore::Semaphore(PGraphics graphics)
     : graphics(graphics)
 {
@@ -65,7 +23,7 @@ Semaphore::~Semaphore()
 
 Fence::Fence(PGraphics graphics)
     : graphics(graphics)
-    , signaled(false)
+    , signaled("Fence")
 {
     VkFenceCreateInfo info =
         init::FenceCreateInfo(0);
@@ -87,7 +45,7 @@ bool Fence::isSignaled()
     switch (res)
     {
     case VK_SUCCESS:
-        signaled = true;
+        signaled.raise();
         return true;
     case VK_NOT_READY:
         break;
@@ -102,7 +60,7 @@ void Fence::reset()
     if (signaled)
     {
         vkResetFences(graphics->getDevice(), 1, &fence);
-        signaled = false;
+        signaled.reset();
     }
 }
 
@@ -113,7 +71,7 @@ void Fence::wait(uint32 timeout)
     switch (r)
     {
     case VK_SUCCESS:
-        signaled = true;
+        signaled.raise();
         break;
     case VK_TIMEOUT:
         break;
@@ -122,6 +80,11 @@ void Fence::wait(uint32 timeout)
         break;
     }
 }
+Event Fence::asyncWait() const
+{
+    return signaled;
+}
+
 
 VertexDeclaration::VertexDeclaration(const Array<Gfx::VertexElement>& elementList) 
     : elementList(elementList)

src/Engine/Graphics/Vulkan/VulkanGraphicsResources.h

@@ -2,6 +2,8 @@
 #include <vulkan/vulkan.h>
 #include <functional>
 #include "Graphics/GraphicsResources.h"
+#include "VulkanAllocator.h"
+#include "ThreadPool.h"
 
 namespace Seele
 {
@@ -41,6 +43,7 @@ public:
         return fence;
     }
     void wait(uint32 timeout);
+    Event asyncWait() const;
     bool operator<(const Fence &other) const
     {
         return fence < other.fence;
@@ -48,7 +51,7 @@ public:
 
 private:
     PGraphics graphics;
-	bool signaled;
+    Event signaled;
     VkFence fence;
 };
 DEFINE_REF(Fence)
@@ -64,27 +67,6 @@ private:
 };
 DEFINE_REF(VertexDeclaration)
 
-class QueueOwnedResourceDeletion
-{
-public:
-	QueueOwnedResourceDeletion();
-	virtual ~QueueOwnedResourceDeletion();
-	static void addPendingDelete(PCmdBuffer fence, std::function<void()> function);
-
-private:
-	std::thread worker;
-	static volatile bool running;
-	static void run();
-	struct PendingItem
-	{
-		PCmdBuffer cmdBuffer;
-		std::function<void()> func;
-	};
-	static std::mutex mutex;
-	static std::condition_variable cv;
-	static List<PendingItem> deletionQueue;
-};
-
 class ShaderBuffer
 {
 public:

src/Engine/Graphics/Vulkan/VulkanPipelineCache.cpp

@@ -29,6 +29,7 @@ PipelineCache::PipelineCache(PGraphics graphics, const std::string& cacheFilePat
         stream.read((char*)cacheData.data(), fileSize);
         cacheCreateInfo.initialDataSize = fileSize;
         cacheCreateInfo.pInitialData = cacheData.data();
+        std::cout << "Loaded " << fileSize << " bytes from pipeline cache" << std::endl;
     }
     VK_CHECK(vkCreatePipelineCache(graphics->getDevice(), &cacheCreateInfo, nullptr, &cache));
 }
@@ -44,6 +45,7 @@ PipelineCache::~PipelineCache()
     stream.flush();
     stream.close();
     vkDestroyPipelineCache(graphics->getDevice(), cache, nullptr);
+    std::cout << "Written " << cacheSize << " bytes to cache" << std::endl;
 }
 
 struct PipelineCreateHashStruct
@@ -321,6 +323,7 @@ PGraphicsPipeline PipelineCache::createPipeline(const GraphicsPipelineCreateInfo
     uint32 hash = crc.checksum();
     VkPipeline pipelineHandle;
 
+    std::unique_lock lock(createdPipelinesLock);
     auto foundPipeline = createdPipelines.find(hash);
     if (foundPipeline != createdPipelines.end())
     {

src/Engine/Graphics/Vulkan/VulkanPipelineCache.h

@@ -16,6 +16,7 @@ private:
     VkPipelineCache cache;
     PGraphics graphics;
     std::string cacheFile;
+    std::mutex createdPipelinesLock;
     Map<uint32, VkPipeline> createdPipelines;
 };
 DEFINE_REF(PipelineCache)

src/Engine/Graphics/Vulkan/VulkanTexture.cpp

@@ -4,6 +4,7 @@
 #include "VulkanGraphicsEnums.h"
 #include "VulkanAllocator.h"
 #include "VulkanCommandBuffer.h"
+#include "ThreadPool.h"
 #include <math.h>
 
 using namespace Seele;
@@ -161,13 +162,17 @@ TextureHandle::TextureHandle(PGraphics graphics, VkImageViewType viewType,
 
 TextureHandle::~TextureHandle()
 {
-    auto &deletionQueue = graphics->getDeletionQueue();
     auto cmdBuffer = graphics->getQueueCommands(currentOwner)->getCommands();
     VkDevice device = graphics->getDevice();
     VkImageView view = defaultView;
     VkImage img = image;
-    deletionQueue.addPendingDelete(cmdBuffer, [device, view]() { vkDestroyImageView(device, view, nullptr); });
+    auto deletionLambda = [cmdBuffer, device, view, img]() -> Job
-    deletionQueue.addPendingDelete(cmdBuffer, [device, img]() { vkDestroyImage(device, img, nullptr); });
+    {
+        vkDestroyImageView(device, view, nullptr);
+        vkDestroyImage(device, img, nullptr);
+        co_return;
+    };
+    deletionLambda();
 }
 
 TextureHandle* TextureBase::cast(Gfx::PTexture texture) 

src/Engine/Material/MaterialAsset.cpp

@@ -1,6 +1,7 @@
 #include "MaterialAsset.h"
 #include "Window/WindowManager.h"
 #include "Asset/AssetRegistry.h"
+#include "Asset/TextureAsset.h"
 #include "BRDF.h"
 #include <nlohmann/json.hpp>
 #include <sstream>

src/Engine/MinimalEngine.h

@@ -38,13 +38,9 @@ public:
         : handle(ptr)
         , deleter(std::move(deleter))
         , refCount(1)
-    {
-        registeredObjects[ptr] = this;
-    }
-    inline RefObject(const RefObject &rhs)
-        : handle(rhs.handle), refCount(rhs.refCount)
     {
     }
+    RefObject(const RefObject &rhs) = delete;
     RefObject(RefObject &&rhs)
         : handle(std::move(rhs.handle)), refCount(std::move(rhs.refCount))
     {
@@ -56,18 +52,11 @@ public:
             registeredObjects.erase(handle);
         }
 //    #pragma warning( disable: 4150)
-        //deleter(handle);
+        deleter(handle);
+        handle = nullptr;
 //    #pragma warning( default: 4150)
     }
-    RefObject &operator=(const RefObject &rhs)
+    RefObject &operator=(const RefObject &rhs) = delete;
-    {
-        if (*this != rhs)
-        {
-            handle = rhs.handle;
-            refCount = rhs.refCount;
-        }
-        return *this;
-    }
     RefObject &operator=(RefObject &&rhs)
     {
         if (*this != rhs)
@@ -131,11 +120,10 @@ public:
         if (registeredObj == registeredEnd)
         {
             object = new RefObject<T, Deleter>(ptr, std::move(deleter));
-            l.unlock();
+            registeredObjects[ptr] = object;
         }
         else
         {		
-            l.unlock();
             object = (RefObject<T, Deleter> *)registeredObj->value;
             object->addRef();
         }

src/Engine/ThreadPool.cpp

@@ -38,6 +38,7 @@ ThreadPool::ThreadPool(uint32 threadCount)
 
 ThreadPool::~ThreadPool()
 {
+    running.store(false);
     for(auto& thread : workers)
     {
         thread.join();
@@ -74,7 +75,7 @@ void ThreadPool::notify(Event* event)
     {
         std::unique_lock lock(jobQueueLock);
         std::unique_lock lock2(waitingLock);
-        while(!waitingJobs.empty())
+        while(!waitingJobs[*event].empty())
         {
             //std::cout << "Waking up job " << job << std::endl;
             jobQueue.add(std::move(waitingJobs[*event].retrieve()));
@@ -90,7 +91,6 @@ void ThreadPool::notify(Event* event)
             mainJobCV.notify_one();
         }
     }
-    event->reset();
 }
 void ThreadPool::threadLoop(const bool mainThread)
 {

src/Engine/ThreadPool.h

@@ -91,6 +91,10 @@ public:
     {
         return *this;
     }
+    operator bool()
+    {
+        return flag->load();
+    }
     
     void raise();
     void reset();

src/Engine/UI/Elements/Element.cpp

@@ -1,4 +1,5 @@
 #include "Element.h"
+#include "UI/System.h"
 
 using namespace Seele;
 using namespace Seele::UI;

src/Engine/UI/System.cpp

@@ -1,4 +1,5 @@
 #include "System.h"
+#include "Elements/Panel.h"
 
 using namespace Seele;
 using namespace Seele::UI;

src/Engine/Window/InspectorView.cpp

@@ -1,4 +1,6 @@
 #include "InspectorView.h"
+#include "Graphics/Graphics.h"
+#include "Scene/Actor/Actor.h"
 #include "Window.h"
 
 using namespace Seele;

src/Engine/Window/SceneView.cpp

@@ -1,6 +1,8 @@
 #include "SceneView.h"
 #include "Scene/Scene.h"
 #include "Window.h"
+#include "Graphics/Graphics.h"
+#include "Asset/MeshAsset.h"
 #include "Asset/AssetRegistry.h"
 #include "Scene/Actor/CameraActor.h"
 #include "Scene/Components/CameraComponent.h"

src/Engine/Window/Window.cpp

@@ -30,6 +30,7 @@ MainJob Window::render()
     for(auto& windowView : views)
     {
         co_await windowView->updateFinished;
+        windowView->updateFinished.reset();
         {
             std::unique_lock lock(windowView->workerMutex);
             windowView->view->prepareRender();