Adding basic timestamps

src/Engine/Graphics/Graphics.h

@@ -32,6 +32,7 @@ DECLARE_REF(RenderCommand)
 DECLARE_REF(ComputeCommand)
 DECLARE_REF(OcclusionQuery)
 DECLARE_REF(PipelineStatisticsQuery)
+DECLARE_REF(TimestampQuery)
 DECLARE_REF(BottomLevelAS)
 DECLARE_REF(TopLevelAS)
 DECLARE_REF(RayGenShader)
@@ -87,8 +88,9 @@ class Graphics {
 
     virtual OVertexInput createVertexInput(VertexInputStateCreateInfo createInfo) = 0;
 
-    virtual Gfx::OOcclusionQuery createOcclusionQuery() = 0;
+    virtual Gfx::OOcclusionQuery createOcclusionQuery(const std::string& name = "") = 0;
-    virtual Gfx::OPipelineStatisticsQuery createPipelineStatisticsQuery() = 0;
+    virtual Gfx::OPipelineStatisticsQuery createPipelineStatisticsQuery(const std::string& name = "") = 0;
+    virtual Gfx::OTimestampQuery createTimestampQuery(uint64 numTimestamps, const std::string& name = "") = 0;
 
     virtual void resolveTexture(PTexture source, PTexture destination) = 0;
     virtual void copyTexture(PTexture src, PTexture dst) = 0;

src/Engine/Graphics/Query.cpp

@@ -13,4 +13,10 @@ PipelineStatisticsQuery::PipelineStatisticsQuery()
 {}
 
 PipelineStatisticsQuery::~PipelineStatisticsQuery()
+{}
+
+TimestampQuery::TimestampQuery()
+{}
+
+TimestampQuery::~TimestampQuery()
 {}

src/Engine/Graphics/Query.h

@@ -1,5 +1,11 @@
 #pragma once
+#include "Containers/Array.h"
+#include "Enums.h"
 #include "MinimalEngine.h"
+#include <ostream>
+#include <chrono>
+#include <fmt/format.h>
+#include <string>
 
 namespace Seele {
 namespace Gfx {
@@ -25,7 +31,15 @@ struct PipelineStatisticsResult {
     uint64 computeShaderInvocations;
     uint64 taskShaderInvocations;
     uint64 meshShaderInvocations;
+    friend std::ostream& operator<<(std::ostream& buf, const PipelineStatisticsResult& res);
 };
+
+static std::ostream& operator<<(std::ostream & buf, const PipelineStatisticsResult& res) {
+    buf << fmt::format("{},{},{},{},{},{},{},{},{},", res.inputAssemblyVertices, res.inputAssemblyPrimitives, res.vertexShaderInvocations,
+                       res.clippingInvocations, res.clippingPrimitives, res.fragmentShaderInvocations, res.computeShaderInvocations, res.taskShaderInvocations,
+                       res.meshShaderInvocations);
+    return buf;
+}
 class PipelineStatisticsQuery {
   public:
     PipelineStatisticsQuery();
@@ -35,5 +49,19 @@ class PipelineStatisticsQuery {
     virtual PipelineStatisticsResult getResults() = 0;
 };
 DEFINE_REF(PipelineStatisticsQuery)
+struct Timestamp {
+    std::string name;
+    std::chrono::nanoseconds time;
+};
+class TimestampQuery {
+  public:
+    TimestampQuery();
+    virtual ~TimestampQuery();
+    virtual void begin() = 0;
+    virtual void write(SePipelineStageFlagBits stage, const std::string& name = "") = 0;
+    virtual void end() = 0;
+    virtual Array<Timestamp> getResults() = 0;
+};
+DEFINE_REF(TimestampQuery)
 } // namespace Gfx
 } // namespace Seele

src/Engine/Graphics/RenderPass/BasePass.cpp

@@ -92,6 +92,7 @@ void BasePass::render() {
     transparentCulling->writeChanges();
 
     query->beginQuery();
+    timestamps->write(Gfx::SE_PIPELINE_STAGE_TOP_OF_PIPE_BIT, "BASEPASS");
     graphics->beginRenderPass(renderPass);
     Array<Gfx::ORenderCommand> commands;
 
@@ -294,6 +295,8 @@ void BasePass::render() {
 
     graphics->endRenderPass();
     query->endQuery();
+    timestamps->write(Gfx::SE_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, "END");
+    timestamps->end();
 }
 
 void BasePass::endFrame() {}
@@ -316,11 +319,12 @@ void BasePass::publishOutputs() {
                                     Gfx::SE_ATTACHMENT_LOAD_OP_CLEAR, Gfx::SE_ATTACHMENT_STORE_OP_STORE);
     resources->registerRenderPassOutput("BASEPASS_DEPTH", depthAttachment);
 
-    query = graphics->createPipelineStatisticsQuery();
+    query = graphics->createPipelineStatisticsQuery("BasePassPipelineStatistics");
     resources->registerQueryOutput("BASEPASS_QUERY", query);
 }
 
 void BasePass::createRenderPass() {
+    timestamps = resources->requestTimestampQuery("TIMESTAMP");
     cullingBuffer = resources->requestBuffer("CULLINGBUFFER");
 
     Gfx::RenderTargetLayout layout = Gfx::RenderTargetLayout{

src/Engine/Graphics/RenderPass/BasePass.h

@@ -40,6 +40,7 @@ class BasePass : public RenderPass {
     Gfx::ODescriptorLayout lightCullingLayout;
 
     Gfx::OPipelineStatisticsQuery query;
+    Gfx::PTimestampQuery timestamps;
 
     Gfx::PShaderBuffer cullingBuffer;
 };

src/Engine/Graphics/RenderPass/CachedDepthPass.cpp

@@ -47,6 +47,8 @@ void CachedDepthPass::beginFrame(const Component::Camera& cam) { RenderPass::beg
 
 void CachedDepthPass::render() {
     query->beginQuery();
+    timestamps->begin();
+    timestamps->write(Gfx::SE_PIPELINE_STAGE_TOP_OF_PIPE_BIT, "CACHED");
     graphics->beginRenderPass(renderPass);
     Array<Gfx::ORenderCommand> commands;
 
@@ -165,8 +167,11 @@ void CachedDepthPass::publishOutputs() {
         Gfx::RenderTargetAttachment(visibilityBuffer, Gfx::SE_IMAGE_LAYOUT_UNDEFINED, Gfx::SE_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
                                     Gfx::SE_ATTACHMENT_LOAD_OP_CLEAR, Gfx::SE_ATTACHMENT_STORE_OP_STORE);
     resources->registerRenderPassOutput("VISIBILITY", visibilityAttachment);
-    query = graphics->createPipelineStatisticsQuery();
+    query = graphics->createPipelineStatisticsQuery("CachedPipelineStatistics");
     resources->registerQueryOutput("CACHED_QUERY", query);
+
+    timestamps = graphics->createTimestampQuery(7, "Timestamps");
+    resources->registerTimestampQueryOutput("TIMESTAMP", timestamps);
 }
 
 void CachedDepthPass::createRenderPass() {

src/Engine/Graphics/RenderPass/CachedDepthPass.h

@@ -22,6 +22,7 @@ class CachedDepthPass : public RenderPass {
     Gfx::OTexture2D visibilityBuffer;
     Gfx::OPipelineLayout depthPrepassLayout;
     Gfx::OPipelineStatisticsQuery query;
+    Gfx::OTimestampQuery timestamps;
 
     Gfx::PShaderBuffer cullingBuffer;
 };

src/Engine/Graphics/RenderPass/DepthCullingPass.cpp

@@ -71,6 +71,7 @@ DepthCullingPass::~DepthCullingPass() {}
 void DepthCullingPass::beginFrame(const Component::Camera& cam) { RenderPass::beginFrame(cam); }
 
 void DepthCullingPass::render() {
+    query->beginQuery();
     depthAttachment.getTexture()->changeLayout(Gfx::SE_IMAGE_LAYOUT_GENERAL, Gfx::SE_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
                                                Gfx::SE_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT, Gfx::SE_ACCESS_TRANSFER_READ_BIT,
                                                Gfx::SE_PIPELINE_STAGE_TRANSFER_BIT);
@@ -80,6 +81,7 @@ void DepthCullingPass::render() {
     set->updateBuffer(1, depthMipBuffer);
     set->writeChanges();
 
+    timestamps->write(Gfx::SE_PIPELINE_STAGE_TOP_OF_PIPE_BIT, "DEPTHMIP");
     Gfx::OComputeCommand computeCommand = graphics->createComputeCommand("DepthMipGenCommand");
     computeCommand->bindPipeline(depthInitialReduce);
     computeCommand->bindDescriptor({viewParamsSet, set});
@@ -116,7 +118,7 @@ void DepthCullingPass::render() {
         Gfx::SE_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | Gfx::SE_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
         Gfx::SE_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT);
 
-    query->beginQuery();
+    timestamps->write(Gfx::SE_PIPELINE_STAGE_TOP_OF_PIPE_BIT, "DEPTHCULL");
     graphics->beginRenderPass(renderPass);
     Array<Gfx::ORenderCommand> commands;
 
@@ -262,11 +264,12 @@ void DepthCullingPass::publishOutputs() {
 
     depthMipGen = graphics->createComputePipeline(pipelineCreateInfo);
 
-    query = graphics->createPipelineStatisticsQuery();
+    query = graphics->createPipelineStatisticsQuery("DepthPipelineStatistics");
     resources->registerQueryOutput("DEPTH_QUERY", query);
 }
 
 void DepthCullingPass::createRenderPass() {
+    timestamps = resources->requestTimestampQuery("TIMESTAMP");
     cullingBuffer = resources->requestBuffer("CULLINGBUFFER");
 
     depthAttachment = resources->requestRenderTarget("DEPTHPREPASS_DEPTH");

src/Engine/Graphics/RenderPass/DepthCullingPass.h

@@ -33,6 +33,7 @@ class DepthCullingPass : public RenderPass {
     Gfx::ODescriptorLayout depthAttachmentLayout;
     Gfx::OPipelineLayout depthCullingLayout;
     Gfx::OPipelineStatisticsQuery query;
+    Gfx::PTimestampQuery timestamps;
 
     Gfx::OPipelineLayout depthComputeLayout;
     Gfx::OComputeShader depthInitialReduceShader;

src/Engine/Graphics/RenderPass/LightCullingPass.cpp

@@ -41,6 +41,7 @@ void LightCullingPass::beginFrame(const Component::Camera& cam) {
 
 void LightCullingPass::render() {
     query->beginQuery();
+    timestamps->write(Gfx::SE_PIPELINE_STAGE_TOP_OF_PIPE_BIT, "LIGHTCULL");
     depthAttachment->transferOwnership(Gfx::QueueType::COMPUTE);
     cullingDescriptorSet->updateTexture(0, depthAttachment);
     cullingDescriptorSet->updateBuffer(1, oLightIndexCounter);
@@ -211,12 +212,14 @@ void LightCullingPass::publishOutputs() {
 
     resources->registerTextureOutput("LIGHTCULLING_OLIGHTGRID", Gfx::PTexture2D(oLightGrid));
     resources->registerTextureOutput("LIGHTCULLING_TLIGHTGRID", Gfx::PTexture2D(tLightGrid));
+
+    query = graphics->createPipelineStatisticsQuery("LightCullPipelineStatistics");
+    resources->registerQueryOutput("LIGHTCULL_QUERY", query);
 }
 
 void LightCullingPass::createRenderPass() {
     depthAttachment = resources->requestRenderTarget("DEPTHPREPASS_DEPTH").getTexture();
-    query = graphics->createPipelineStatisticsQuery();
+    timestamps = resources->requestTimestampQuery("TIMESTAMP");
-    resources->registerQueryOutput("LIGHTCULL_QUERY", query);
 }
 
 void LightCullingPass::setupFrustums() {

src/Engine/Graphics/RenderPass/LightCullingPass.h

@@ -65,6 +65,7 @@ class LightCullingPass : public RenderPass {
     Gfx::PComputePipeline cullingPipeline;
     Gfx::PComputePipeline cullingEnabledPipeline;
     Gfx::OPipelineStatisticsQuery query;
+    Gfx::PTimestampQuery timestamps;
 };
 DEFINE_REF(LightCullingPass)
 } // namespace Seele

src/Engine/Graphics/RenderPass/RenderGraphResources.cpp

@@ -21,6 +21,8 @@ Gfx::PUniformBuffer RenderGraphResources::requestUniform(const std::string& outp
 
 Gfx::PPipelineStatisticsQuery RenderGraphResources::requestQuery(const std::string& outputName) { return registeredQueries.at(outputName); }
 
+Gfx::PTimestampQuery Seele::RenderGraphResources::requestTimestampQuery(const std::string& outputName) { return registeredTimestamps.at(outputName); }
+
 void RenderGraphResources::registerRenderPassOutput(const std::string& outputName, Gfx::RenderTargetAttachment attachment) {
     registeredAttachments[outputName] = attachment;
 }
@@ -38,4 +40,8 @@ void RenderGraphResources::registerUniformOutput(const std::string& outputName,
 
 void RenderGraphResources::registerQueryOutput(const std::string& outputName, Gfx::PPipelineStatisticsQuery query) {
     registeredQueries[outputName] = query;
-}
+}
+
+void Seele::RenderGraphResources::registerTimestampQueryOutput(const std::string& outputName, Gfx::PTimestampQuery query) {
+    registeredTimestamps[outputName] = query;
+}

src/Engine/Graphics/RenderPass/RenderGraphResources.h

@@ -18,11 +18,13 @@ class RenderGraphResources {
     Gfx::PShaderBuffer requestBuffer(const std::string& outputName);
     Gfx::PUniformBuffer requestUniform(const std::string& outputName);
     Gfx::PPipelineStatisticsQuery requestQuery(const std::string& outputName);
+    Gfx::PTimestampQuery requestTimestampQuery(const std::string& outputName);
     void registerRenderPassOutput(const std::string& outputName, Gfx::RenderTargetAttachment attachment);
     void registerTextureOutput(const std::string& outputName, Gfx::PTexture buffer);
     void registerBufferOutput(const std::string& outputName, Gfx::PShaderBuffer buffer);
     void registerUniformOutput(const std::string& outputName, Gfx::PUniformBuffer buffer);
     void registerQueryOutput(const std::string& outputName, Gfx::PPipelineStatisticsQuery query);
+    void registerTimestampQueryOutput(const std::string& outputName, Gfx::PTimestampQuery query);
 
   protected:
     Map<std::string, Gfx::RenderTargetAttachment> registeredAttachments;
@@ -30,6 +32,7 @@ class RenderGraphResources {
     Map<std::string, Gfx::PShaderBuffer> registeredBuffers;
     Map<std::string, Gfx::PUniformBuffer> registeredUniforms;
     Map<std::string, Gfx::PPipelineStatisticsQuery> registeredQueries;
+    Map<std::string, Gfx::PTimestampQuery> registeredTimestamps;
 };
 DEFINE_REF(RenderGraphResources)
 } // namespace Seele

src/Engine/Graphics/RenderPass/VisibilityPass.cpp

@@ -28,6 +28,7 @@ void VisibilityPass::render() {
     visibilitySet->writeChanges();
 
     query->beginQuery();
+    timestamps->write(Gfx::SE_PIPELINE_STAGE_TOP_OF_PIPE_BIT, "VISIBILITY");
     Gfx::OComputeCommand command = graphics->createComputeCommand("VisibilityCommand");
     command->bindPipeline(visibilityPipeline);
     command->bindDescriptor({viewParamsSet, visibilitySet});
@@ -85,7 +86,8 @@ void VisibilityPass::publishOutputs() {
     });
     resources->registerBufferOutput("CULLINGBUFFER", cullingBuffer);
 
-    query = graphics->createPipelineStatisticsQuery();
+    query = graphics->createPipelineStatisticsQuery("VisibilityPipelineStatistics");
+    timestamps = resources->requestTimestampQuery("TIMESTAMP");
     resources->registerQueryOutput("VISIBILITY_QUERY", query);
 }
 

src/Engine/Graphics/RenderPass/VisibilityPass.h

@@ -24,6 +24,7 @@ class VisibilityPass : public RenderPass {
     Gfx::OComputeShader visibilityShader;
     Gfx::PComputePipeline visibilityPipeline;
     Gfx::OPipelineStatisticsQuery query;
+    Gfx::PTimestampQuery timestamps;
 
     // Holds culling information for every meshlet for each instance
     Gfx::OShaderBuffer cullingBuffer;

src/Engine/Graphics/Vulkan/Graphics.cpp

@@ -239,9 +239,15 @@ Gfx::OPipelineLayout Graphics::createPipelineLayout(const std::string& name, Gfx
 
 Gfx::OVertexInput Graphics::createVertexInput(VertexInputStateCreateInfo createInfo) { return new VertexInput(createInfo); }
 
-Gfx::OOcclusionQuery Graphics::createOcclusionQuery() { return new OcclusionQuery(this); }
+Gfx::OOcclusionQuery Graphics::createOcclusionQuery(const std::string& name) { return new OcclusionQuery(this, name); }
 
-Gfx::OPipelineStatisticsQuery Graphics::createPipelineStatisticsQuery() { return new PipelineStatisticsQuery(this); }
+Gfx::OPipelineStatisticsQuery Graphics::createPipelineStatisticsQuery(const std::string& name) {
+    return new PipelineStatisticsQuery(this, name);
+}
+
+Gfx::OTimestampQuery Graphics::createTimestampQuery(uint64 numTimestamps, const std::string& name) {
+    return new TimestampQuery(this, name, numTimestamps);
+}
 
 void Graphics::resolveTexture(Gfx::PTexture source, Gfx::PTexture destination) {
     PTextureBase sourceTex = source.cast<TextureBase>();
@@ -642,7 +648,6 @@ void Graphics::createDevice(GraphicsInitializer initializer) {
             *currentPriority++ = 1.0f;
         }
     }
-
     if (supportMeshShading()) {
         initializer.deviceExtensions.add(VK_EXT_MESH_SHADER_EXTENSION_NAME);
     }
@@ -683,6 +688,8 @@ void Graphics::createDevice(GraphicsInitializer initializer) {
     queueMapping.computeFamily = queues[computeQueue]->getFamilyIndex();
     queueMapping.transferFamily = queues[transferQueue]->getFamilyIndex();
 
+    graphicsProps = queueProperties[queueMapping.graphicsFamily];
+
     cmdDrawMeshTasks = (PFN_vkCmdDrawMeshTasksEXT)vkGetDeviceProcAddr(handle, "vkCmdDrawMeshTasksEXT");
     cmdDrawMeshTasksIndirect = (PFN_vkCmdDrawMeshTasksIndirectEXT)vkGetDeviceProcAddr(handle, "vkCmdDrawMeshTasksIndirectEXT");
     setDebugUtilsObjectName = (PFN_vkSetDebugUtilsObjectNameEXT)vkGetInstanceProcAddr(instance, "vkSetDebugUtilsObjectNameEXT");

src/Engine/Graphics/Vulkan/Graphics.h

@@ -18,6 +18,8 @@ class Graphics : public Gfx::Graphics {
     constexpr VkPhysicalDevice getPhysicalDevice() const { return physicalDevice; }
     constexpr VkPhysicalDeviceAccelerationStructurePropertiesKHR getAccelerationProperties() const { return accelerationProperties; }
     constexpr VkPhysicalDeviceRayTracingPipelinePropertiesKHR getRayTracingProperties() const { return rayTracingProperties; }
+    constexpr float getTimestampPeriod() const { return props.properties.limits.timestampPeriod; }
+    constexpr uint64 getTimestampValidBits() const { return graphicsProps.timestampValidBits; }
 
     PCommandPool getQueueCommands(Gfx::QueueType queueType);
     PCommandPool getGraphicsCommands();
@@ -67,8 +69,9 @@ class Graphics : public Gfx::Graphics {
 
     virtual Gfx::OVertexInput createVertexInput(VertexInputStateCreateInfo createInfo) override;
 
-    virtual Gfx::OOcclusionQuery createOcclusionQuery() override;
+    virtual Gfx::OOcclusionQuery createOcclusionQuery(const std::string& name = "") override;
-    virtual Gfx::OPipelineStatisticsQuery createPipelineStatisticsQuery() override;
+    virtual Gfx::OPipelineStatisticsQuery createPipelineStatisticsQuery(const std::string& name = "") override;
+    virtual Gfx::OTimestampQuery createTimestampQuery(uint64 numTimestamps, const std::string& name = "") override;
 
     virtual void resolveTexture(Gfx::PTexture source, Gfx::PTexture destination) override;
     virtual void copyTexture(Gfx::PTexture src, Gfx::PTexture dst) override;
@@ -104,6 +107,7 @@ class Graphics : public Gfx::Graphics {
     thread_local static PCommandPool transferCommands;
     std::mutex poolLock;
     Array<OCommandPool> pools;
+    VkQueueFamilyProperties graphicsProps;
     VkPhysicalDeviceProperties2 props;
     VkPhysicalDeviceFeatures2 features;
     VkPhysicalDeviceVulkan12Features features12;

src/Engine/Graphics/Vulkan/Query.cpp

@@ -10,7 +10,8 @@
 using namespace Seele;
 using namespace Seele::Vulkan;
 
-QueryPool::QueryPool(PGraphics graphics, VkQueryType type, VkQueryPipelineStatisticFlags flags, uint32 resultsStride, uint32 numBuffered)
+QueryPool::QueryPool(PGraphics graphics, VkQueryType type, VkQueryPipelineStatisticFlags flags, uint32 resultsStride, uint32 numBuffered,
+                     const std::string& name)
     : graphics(graphics), flags(flags), numQueries(numBuffered), resultsStride(resultsStride) {
     VkQueryPoolCreateInfo info = {
         .sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,
@@ -21,58 +22,49 @@ QueryPool::QueryPool(PGraphics graphics, VkQueryType type, VkQueryPipelineStatis
         .pipelineStatistics = flags,
     };
     VK_CHECK(vkCreateQueryPool(graphics->getDevice(), &info, nullptr, &handle));
-    //VkBufferCreateInfo bufferInfo = {
+    VkDebugUtilsObjectNameInfoEXT nameInfo = {
-    //    .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
+        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT,
-    //    .pNext = nullptr,
+        .pNext = nullptr,
-    //    .flags = 0,
+        .objectType = VK_OBJECT_TYPE_QUERY_POOL,
-    //    .size = (resultsStride + 1) * numQueries * sizeof(uint64),
+        .objectHandle = (uint64)handle,
-    //    .usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
+        .pObjectName = name.c_str(),
-    //};
+    };
-    //VmaAllocationCreateInfo allocInfo = {
+    vkSetDebugUtilsObjectNameEXT(graphics->getDevice(), &nameInfo);
-    //    .flags = VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT,
+    PCommand cmd = graphics->getGraphicsCommands()->getCommands();
-    //    .usage = VMA_MEMORY_USAGE_AUTO,
+    vkCmdResetQueryPool(cmd->getHandle(), handle, head, numQueries);
-    //    .requiredFlags = VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
-    //};
-    //resultsAlloc = new BufferAllocation(graphics, "QueryResults", bufferInfo, allocInfo, Gfx::QueueType::GRAPHICS);
-    //VK_CHECK(vmaMapMemory(graphics->getAllocator(), resultsAlloc->allocation, (void**) & resultsPtr));
 }
 
 QueryPool::~QueryPool() { vkDestroyQueryPool(graphics->getDevice(), handle, nullptr); }
 
 void QueryPool::begin() {
-    vkCmdResetQueryPool(graphics->getGraphicsCommands()->getCommands()->getHandle(), handle, currentQuery, 1);
+    PCommand cmd = graphics->getGraphicsCommands()->getCommands();
-    vkCmdBeginQuery(graphics->getGraphicsCommands()->getCommands()->getHandle(), handle, currentQuery, 0);
+    vkCmdResetQueryPool(cmd->getHandle(), handle, head, 1);
-    graphics->getGraphicsCommands()->getCommands()->setPipelineStatisticsFlags(flags);
+    vkCmdBeginQuery(cmd->getHandle(), handle, head, 0);
+    cmd->setPipelineStatisticsFlags(flags);
 }
 
 void QueryPool::end() {
-    vkCmdEndQuery(graphics->getGraphicsCommands()->getCommands()->getHandle(), handle, currentQuery);
+    PCommand cmd = graphics->getGraphicsCommands()->getCommands();
-    //vkCmdCopyQueryPoolResults(graphics->getGraphicsCommands()->getCommands()->getHandle(), handle, currentQuery, 1, resultsAlloc->buffer,
+    vkCmdEndQuery(cmd->getHandle(), handle, head);
-    //                          sizeof(uint64) * currentQuery, resultsStride + sizeof(uint64),
-    //                          VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
     graphics->getGraphicsCommands()->submitCommands();
     std::unique_lock l(queryMutex);
-    currentQuery = (currentQuery + 1) % numQueries;
+    head = (head + 1) % numQueries;
     queryCV.notify_all();
 }
 
 void QueryPool::getQueryResults(Array<uint64>& results) {
-    //uint64 numInts = resultsStride / sizeof(uint64);
+    while (tail == head) {
-    while (currentQuery == pendingQuery)
-    {
         std::unique_lock l(queryMutex);
         queryCV.wait(l);
     }
-    results.resize(resultsStride/ sizeof(uint64));
+    results.resize(resultsStride / sizeof(uint64));
-    vkGetQueryPoolResults(graphics->getDevice(), handle, pendingQuery, 1, resultsStride, results.data(), resultsStride,
+    vkGetQueryPoolResults(graphics->getDevice(), handle, tail, 1, resultsStride, results.data(), resultsStride,
                           VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_64_BIT);
-    //uint64* currentPtr = resultsPtr + (pendingQuery * (numInts + 1));
+    tail = (tail + 1) % numQueries;
-    //std::cout << pendingQuery << *(currentPtr + numInts) << std::endl;
-    //std::memcpy(results.data(), currentPtr, resultsStride);
-    pendingQuery = (pendingQuery + 1) % numQueries;
 }
 
-OcclusionQuery::OcclusionQuery(PGraphics graphics) : QueryPool(graphics, VK_QUERY_TYPE_OCCLUSION, 0, sizeof(uint64), 16) {}
+OcclusionQuery::OcclusionQuery(PGraphics graphics, const std::string& name)
+    : QueryPool(graphics, VK_QUERY_TYPE_OCCLUSION, 0, sizeof(uint64), 16, name) {}
 
 OcclusionQuery::~OcclusionQuery() {}
 
@@ -88,7 +80,7 @@ Gfx::OcclusionResult OcclusionQuery::getResults() {
     };
 }
 
-PipelineStatisticsQuery::PipelineStatisticsQuery(PGraphics graphics)
+PipelineStatisticsQuery::PipelineStatisticsQuery(PGraphics graphics, const std::string& name)
     : QueryPool(graphics, VK_QUERY_TYPE_PIPELINE_STATISTICS,
                 VK_QUERY_PIPELINE_STATISTIC_INPUT_ASSEMBLY_VERTICES_BIT | VK_QUERY_PIPELINE_STATISTIC_INPUT_ASSEMBLY_PRIMITIVES_BIT |
                     VK_QUERY_PIPELINE_STATISTIC_VERTEX_SHADER_INVOCATIONS_BIT | VK_QUERY_PIPELINE_STATISTIC_CLIPPING_INVOCATIONS_BIT |
@@ -96,7 +88,7 @@ PipelineStatisticsQuery::PipelineStatisticsQuery(PGraphics graphics)
                     VK_QUERY_PIPELINE_STATISTIC_COMPUTE_SHADER_INVOCATIONS_BIT |
                     VK_QUERY_PIPELINE_STATISTIC_TASK_SHADER_INVOCATIONS_BIT_EXT |
                     VK_QUERY_PIPELINE_STATISTIC_MESH_SHADER_INVOCATIONS_BIT_EXT,
-                sizeof(PipelineStatisticsQuery), 16) {}
+                sizeof(PipelineStatisticsQuery), 128, name) {}
 
 PipelineStatisticsQuery::~PipelineStatisticsQuery() {}
 
@@ -118,4 +110,55 @@ Gfx::PipelineStatisticsResult PipelineStatisticsQuery::getResults() {
         .taskShaderInvocations = result[7],
         .meshShaderInvocations = result[8],
     };
+}
+
+TimestampQuery::TimestampQuery(PGraphics graphics, const std::string& name, uint32 numTimestamps)
+    : QueryPool(graphics, VK_QUERY_TYPE_TIMESTAMP, 0, sizeof(uint64), 512 * numTimestamps, name), numTimestamps(numTimestamps) {
+    pendingTimestamps.resize(numQueries);
+}
+
+TimestampQuery::~TimestampQuery() {}
+
+void TimestampQuery::begin() {
+    currentTimestamp = 0;
+}
+
+void TimestampQuery::write(Gfx::SePipelineStageFlagBits stage, const std::string& name) {
+    PCommand cmd = graphics->getGraphicsCommands()->getCommands();
+    uint32 queryIndex = (head * numTimestamps) + currentTimestamp;
+    vkCmdResetQueryPool(cmd->getHandle(), handle, queryIndex, 1);
+    vkCmdWriteTimestamp(cmd->getHandle(), cast(stage), handle, queryIndex);
+    pendingTimestamps[queryIndex] = name;
+    currentTimestamp++;
+}
+
+void TimestampQuery::end() {
+    std::unique_lock l(queryMutex);
+    head = (head + 1) % 512;
+    queryCV.notify_all();
+}
+
+Array<Gfx::Timestamp> TimestampQuery::getResults() {
+    while (head == tail) {
+        std::unique_lock l(queryMutex);
+        queryCV.wait(l);
+    }
+    Array<uint64> results(numTimestamps);
+    uint32 firstQuery = (tail * numTimestamps);
+    vkGetQueryPoolResults(graphics->getDevice(), handle, firstQuery, numTimestamps, numTimestamps * sizeof(uint64), results.data(),
+                          resultsStride, VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_64_BIT);
+    tail = (tail + 1) % 512;
+    Array<Gfx::Timestamp> res;
+    for (uint64 i = 0; i < numTimestamps; ++i) {
+        if (results[i] < lastMeasure)
+        {
+            wrapping += 1ull << graphics->getTimestampValidBits();
+        }
+        lastMeasure = results[i];
+        res.add(Gfx::Timestamp{
+            .name = pendingTimestamps[firstQuery + i],
+            .time = std::chrono::nanoseconds(uint64((results[i] + wrapping) * graphics->getTimestampPeriod())),
+        });
+    }
+    return res;
 }

src/Engine/Graphics/Vulkan/Query.h

@@ -1,13 +1,14 @@
 #pragma once
+#include "Buffer.h"
 #include "Graphics.h"
 #include "Graphics/Query.h"
-#include "Buffer.h"
 
 namespace Seele {
 namespace Vulkan {
 class QueryPool {
   public:
-    QueryPool(PGraphics graphics, VkQueryType type, VkQueryPipelineStatisticFlags flags, uint32 resultsStride, uint32 numBuffered);
+    QueryPool(PGraphics graphics, VkQueryType type, VkQueryPipelineStatisticFlags flags, uint32 resultsStride, uint32 numBuffered,
+              const std::string& name);
     virtual ~QueryPool();
     void begin();
     void end();
@@ -18,10 +19,9 @@ class QueryPool {
     PGraphics graphics;
     VkQueryPool handle;
     VkQueryPipelineStatisticFlags flags;
-    OBufferAllocation resultsAlloc;
+    // ring buffer
-    uint64* resultsPtr;
+    uint64 head = 0;
-    uint32 pendingQuery = 0;
+    uint64 tail = 0;
-    uint32 currentQuery = 0;
     uint32 numQueries;
     uint32 resultsStride;
     std::mutex queryMutex;
@@ -29,7 +29,7 @@ class QueryPool {
 };
 class OcclusionQuery : public Gfx::OcclusionQuery, public QueryPool {
   public:
-    OcclusionQuery(PGraphics graphics);
+    OcclusionQuery(PGraphics graphics, const std::string& name);
     virtual ~OcclusionQuery();
     virtual void beginQuery() override;
     virtual void endQuery() override;
@@ -38,7 +38,7 @@ class OcclusionQuery : public Gfx::OcclusionQuery, public QueryPool {
 DEFINE_REF(OcclusionQuery)
 class PipelineStatisticsQuery : public Gfx::PipelineStatisticsQuery, public QueryPool {
   public:
-    PipelineStatisticsQuery(PGraphics graphics);
+    PipelineStatisticsQuery(PGraphics graphics, const std::string& name);
     virtual ~PipelineStatisticsQuery();
     virtual void beginQuery() override;
     virtual void endQuery() override;
@@ -46,5 +46,22 @@ class PipelineStatisticsQuery : public Gfx::PipelineStatisticsQuery, public Quer
 };
 DEFINE_REF(PipelineStatisticsQuery)
 
+class TimestampQuery : public Gfx::TimestampQuery, public QueryPool {
+  public:
+    TimestampQuery(PGraphics graphics, const std::string& name, uint32 numTimestamps);
+    virtual ~TimestampQuery();
+    virtual void begin() override;
+    virtual void write(Gfx::SePipelineStageFlagBits stage, const std::string& name = "") override;
+    virtual void end() override;
+    virtual Array<Gfx::Timestamp> getResults() override;
+
+  private:
+    uint64 wrapping = 0;
+    uint64 lastMeasure = 0;
+    uint32 numTimestamps = 0;
+    uint32 currentTimestamp = 0;
+    Array<std::string> pendingTimestamps;
+};
+DEFINE_REF(TimestampQuery)
 } // namespace Vulkan
 } // namespace Seele

src/Engine/Graphics/Vulkan/Texture.cpp

@@ -258,7 +258,6 @@ void TextureHandle::changeLayout(Gfx::SeImageLayout newLayout, VkAccessFlags src
     PCommandPool commandPool = graphics->getQueueCommands(owner);
     vkCmdPipelineBarrier(commandPool->getCommands()->getHandle(), srcStage, dstStage, 0, 0, nullptr, 0, nullptr, 1, &barrier);
     commandPool->getCommands()->bindResource(PTextureHandle(this));
-    commandPool->submitCommands();
     layout = newLayout;
 }
 

src/Engine/Window/GameView.cpp

@@ -16,6 +16,7 @@
 #include "System/LightGather.h"
 #include "System/MeshUpdater.h"
 #include "Window/Window.h"
+#include <fstream>
 
 using namespace Seele;
 
@@ -42,16 +43,35 @@ GameView::GameView(Gfx::PGraphics graphics, PWindow window, const ViewportCreate
         Gfx::PPipelineStatisticsQuery baseQuery = res->requestQuery("BASEPASS_QUERY");
         Gfx::PPipelineStatisticsQuery lightCullQuery = res->requestQuery("LIGHTCULL_QUERY");
         Gfx::PPipelineStatisticsQuery visibilityQuery = res->requestQuery("VISIBILITY_QUERY");
+        Gfx::PTimestampQuery timeQuery = res->requestTimestampQuery("TIMESTAMP");
+        std::ofstream stats("stats.csv");
+        stats << "RelTime,"
+              << "CachedIAV,CachedIAP,CachedVS,CachedClipInv,CachedClipPrim,CachedFS,CachedCS,CachedTS,CachedMS,"
+              << "DepthIAV,DepthIAP,DepthVS,DepthClipInv,DepthClipPrim,DepthFS,DepthCS,DepthTS,DepthMS,"
+              << "BaseIAV,BaseIAP,BaseVS,BaseClipInv,BaseClipPrim,BaseFS,BaseCS,BaseTS,BaseMS,"
+              << "LightCullIAV,LightCullIAP,LightCullVS,LightCullClipInv,LightCullClipPrim,LightCullFS,LightCullCS,LightCullTS,LightCullMS,"
+              << "VisibilityIAV,VisibilityIAP,VisibilityVS,VisibilityClipInv,VisibilityClipPrim,VisibilityFS,VisibilityCS,VisibilityTS,"
+                 "VisibilityMS,"
+              << "CACHED,MIPGEN,DEPTHCULL,VISIBILITY,LIGHTCULL,BASE" << std::endl;
+        std::chrono::nanoseconds start = std::chrono::nanoseconds(0);
         while (true) {
+            auto timestamps = timeQuery->getResults();
             auto cachedResults = cachedQuery->getResults();
             auto depthResults = depthQuery->getResults();
             auto baseResults = baseQuery->getResults();
             auto lightCullResults = lightCullQuery->getResults();
             auto visiblityResults = visibilityQuery->getResults();
-            std::cout << "Pipeline Stats: "
+            if (start.count() == 0) {
-                      << cachedResults.meshShaderInvocations + depthResults.meshShaderInvocations + baseResults.meshShaderInvocations +
+                start = timestamps[0].time;
-                             lightCullResults.meshShaderInvocations + visiblityResults.meshShaderInvocations
+            }
-                      << std::endl;
+            stats << (timestamps[0].time - start).count() << "," << cachedResults << depthResults << baseResults << lightCullResults
+                  << visiblityResults;
+            stats << fmt::format("{},{},{},{},{},{}", (timestamps[1].time - timestamps[0].time).count(),
+                                 (timestamps[2].time - timestamps[1].time).count(), (timestamps[3].time - timestamps[2].time).count(),
+                                 (timestamps[4].time - timestamps[3].time).count(), (timestamps[5].time - timestamps[4].time).count(),
+                                 (timestamps[6].time - timestamps[5].time).count())
+                  << std::endl;
+            stats.flush();
         }
     });
 }