Edit

kc3-lang/angle/src/tests/perf_tests/VulkanCommandBufferPerf.cpp

Branch :

  • Show log

    Commit

  • Author : Brian Sheedy
    Date : 2019-08-16 14:09:13
    Hash : 2f4a7518
    Message : Refactor perf tests to fix metric/story swapping Refactors the perf tests to fix the issue of metric and story being swapped, which causes issues when trying to convert to histograms. Specifically, does the following: 1. Rolls the version of src/tests/perf_tests/third_party/perf/ to Chromium 476dae823269c8d05b544271af97ad1adb0db8ee 2. Switch to using PerfResultReporter instead of PrintResult directly. 3. Split RenderTestParams::suffix into backend and story; backend is used as part of the metric, while story is used as the story. 4. Remove the "average" metric that was being automatically reported by ANGLEPerfTest, as reported results are automatically averaged. 5. Update the reported metric to more clearly distinguish between test, backend, and metric. It is now name_backend.metric. e.g. DrawCallPerf_vulkan.wall_time. Bug: chromium:923564,chromium:924618 Change-Id: I00cc191407052f23df57dbfa53b6fb088fc26960 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1762360 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: Jamie Madill <jmadill@chromium.org> Reviewed-by: Jonah Ryan-Davis <jonahr@google.com>

  • src/tests/perf_tests/VulkanCommandBufferPerf.cpp
  • // Copyright 2018 The ANGLE Project Authors. All rights reserved.
    // Use of this source code is governed by a BSD-style license that can be
    // found in the LICENSE file.
    //
    // VulkanCommandBufferPerf:
    //   Performance benchmark for Vulkan Primary/Secondary Command Buffer implementations.
    //  Can run just these tests by adding "--gtest_filter=VulkanCommandBufferPerfTest*"
    //   option to angle_white_box_perftests.
    //  When running on Android with run_angle_white_box_perftests, use "-v" option.
    
    #include "ANGLEPerfTest.h"
    #include "common/platform.h"
    #include "test_utils/third_party/vulkan_command_buffer_utils.h"
    
    #if defined(ANDROID)
    #    define NUM_CMD_BUFFERS 1000
    // Android devices tend to be slower so only do 10 frames to avoid timeout
    #    define NUM_FRAMES 10
    #else
    #    define NUM_CMD_BUFFERS 1000
    #    define NUM_FRAMES 100
    #endif
    
    // These are minimal shaders used to submit trivial draw commands to command
    //  buffers so that we can create large batches of cmd buffers with consistent
    //  draw patterns but size/type of cmd buffers can be varied to test cmd buffer
    //  differences across devices.
    constexpr char kVertShaderText[] = R"(
    #version 400
    #extension GL_ARB_separate_shader_objects : enable
    #extension GL_ARB_shading_language_420pack : enable
    layout (std140, binding = 0) uniform bufferVals {
        mat4 mvp;
    } myBufferVals;
    layout (location = 0) in vec4 pos;
    layout (location = 1) in vec4 inColor;
    layout (location = 0) out vec4 outColor;
    void main() {
       outColor = inColor;
       gl_Position = myBufferVals.mvp * pos;
    })";
    
    constexpr char kFragShaderText[] = R"(
    #version 400
    #extension GL_ARB_separate_shader_objects : enable
    #extension GL_ARB_shading_language_420pack : enable
    layout (location = 0) in vec4 color;
    layout (location = 0) out vec4 outColor;
    void main() {
       outColor = color;
    })";
    
    using CommandBufferImpl = void (*)(sample_info &info,
                                       VkClearValue *clear_values,
                                       VkFence drawFence,
                                       VkSemaphore imageAcquiredSemaphore,
                                       int numBuffers);
    
    struct CommandBufferTestParams
    {
        CommandBufferImpl CBImplementation;
        std::string story;
        int frames  = NUM_FRAMES;
        int buffers = NUM_CMD_BUFFERS;
    };
    
    class VulkanCommandBufferPerfTest : public ANGLEPerfTest,
                                        public ::testing::WithParamInterface<CommandBufferTestParams>
    {
      public:
        VulkanCommandBufferPerfTest();
    
        void SetUp() override;
        void TearDown() override;
        void step() override;
    
      private:
        VkClearValue mClearValues[2]        = {};
        VkSemaphore mImageAcquiredSemaphore = VK_NULL_HANDLE;
        VkFence mDrawFence                  = VK_NULL_HANDLE;
    
        VkResult res             = VK_NOT_READY;
        const bool mDepthPresent = true;
        struct sample_info mInfo = {};
        std::string mSampleTitle;
        CommandBufferImpl mCBImplementation = nullptr;
        int mFrames                         = 0;
        int mBuffers                        = 0;
    };
    
    VulkanCommandBufferPerfTest::VulkanCommandBufferPerfTest()
        : ANGLEPerfTest("VulkanCommandBufferPerfTest", "", GetParam().story, GetParam().frames)
    {
        mInfo             = {};
        mSampleTitle      = "Draw Textured Cube";
        mCBImplementation = GetParam().CBImplementation;
        mFrames           = GetParam().frames;
        mBuffers          = GetParam().buffers;
    
    // This test appears to be flaky on multiple platforms.
    #if !defined(ANGLE_PLATFORM_ANDROID)
        // mSkipTest = true;
    #endif  // !defined(ANGLE_PLATFORM_ANDROID)
    }
    
    void VulkanCommandBufferPerfTest::SetUp()
    {
        if (mSkipTest)
        {
            return;
        }
    
        init_global_layer_properties(mInfo);
        init_instance_extension_names(mInfo);
        init_device_extension_names(mInfo);
        init_instance(mInfo, mSampleTitle.c_str());
        init_enumerate_device(mInfo);
        init_window_size(mInfo, 500, 500);
        init_connection(mInfo);
        init_window(mInfo);
        init_swapchain_extension(mInfo);
        init_device(mInfo);
    
        init_command_pool(mInfo, VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT);
        init_command_buffer(mInfo);                   // Primary command buffer to hold secondaries
        init_command_buffer_array(mInfo, mBuffers);   // Array of primary command buffers
        init_command_buffer2_array(mInfo, mBuffers);  // Array containing all secondary buffers
        init_device_queue(mInfo);
        init_swap_chain(mInfo);
        init_depth_buffer(mInfo);
        init_uniform_buffer(mInfo);
        init_descriptor_and_pipeline_layouts(mInfo, false);
        init_renderpass(mInfo, mDepthPresent);
        init_shaders(mInfo, kVertShaderText, kFragShaderText);
        init_framebuffers(mInfo, mDepthPresent);
        init_vertex_buffer(mInfo, g_vb_solid_face_colors_Data, sizeof(g_vb_solid_face_colors_Data),
                           sizeof(g_vb_solid_face_colors_Data[0]), false);
        init_descriptor_pool(mInfo, false);
        init_descriptor_set(mInfo);
        init_pipeline_cache(mInfo);
        init_pipeline(mInfo, mDepthPresent);
    
        mClearValues[0].color.float32[0]     = 0.2f;
        mClearValues[0].color.float32[1]     = 0.2f;
        mClearValues[0].color.float32[2]     = 0.2f;
        mClearValues[0].color.float32[3]     = 0.2f;
        mClearValues[1].depthStencil.depth   = 1.0f;
        mClearValues[1].depthStencil.stencil = 0;
    
        VkSemaphoreCreateInfo imageAcquiredSemaphoreCreateInfo;
        imageAcquiredSemaphoreCreateInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
        imageAcquiredSemaphoreCreateInfo.pNext = NULL;
        imageAcquiredSemaphoreCreateInfo.flags = 0;
        res = vkCreateSemaphore(mInfo.device, &imageAcquiredSemaphoreCreateInfo, NULL,
                                &mImageAcquiredSemaphore);
        ASSERT_EQ(VK_SUCCESS, res);
    
        VkFenceCreateInfo fenceInfo;
        fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
        fenceInfo.pNext = NULL;
        fenceInfo.flags = 0;
        res             = vkCreateFence(mInfo.device, &fenceInfo, NULL, &mDrawFence);
        ASSERT_EQ(VK_SUCCESS, res);
    }
    
    void VulkanCommandBufferPerfTest::step()
    {
        for (int x = 0; x < mFrames; x++)
        {
            mInfo.current_buffer = x % mInfo.swapchainImageCount;
    
            // Get the index of the next available swapchain image:
            res = vkAcquireNextImageKHR(mInfo.device, mInfo.swap_chain, UINT64_MAX,
                                        mImageAcquiredSemaphore, VK_NULL_HANDLE, &mInfo.current_buffer);
            // Deal with the VK_SUBOPTIMAL_KHR and VK_ERROR_OUT_OF_DATE_KHR
            // return codes
            ASSERT_EQ(VK_SUCCESS, res);
            mCBImplementation(mInfo, mClearValues, mDrawFence, mImageAcquiredSemaphore, mBuffers);
        }
    }
    
    void VulkanCommandBufferPerfTest::TearDown()
    {
        if (mSkipTest)
        {
            return;
        }
    
        vkDestroySemaphore(mInfo.device, mImageAcquiredSemaphore, NULL);
        vkDestroyFence(mInfo.device, mDrawFence, NULL);
        destroy_pipeline(mInfo);
        destroy_pipeline_cache(mInfo);
        destroy_descriptor_pool(mInfo);
        destroy_vertex_buffer(mInfo);
        destroy_framebuffers(mInfo);
        destroy_shaders(mInfo);
        destroy_renderpass(mInfo);
        destroy_descriptor_and_pipeline_layouts(mInfo);
        destroy_uniform_buffer(mInfo);
        destroy_depth_buffer(mInfo);
        destroy_swap_chain(mInfo);
        destroy_command_buffer2_array(mInfo, mBuffers);
        destroy_command_buffer_array(mInfo, mBuffers);
        destroy_command_buffer(mInfo);
        destroy_command_pool(mInfo);
        destroy_device(mInfo);
        destroy_window(mInfo);
        destroy_instance(mInfo);
        ANGLEPerfTest::TearDown();
    }
    
    // Common code to present image used by all tests
    void Present(sample_info &info, VkFence drawFence)
    {
        // Now present the image in the window
    
        VkPresentInfoKHR present;
        present.sType              = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
        present.pNext              = NULL;
        present.swapchainCount     = 1;
        present.pSwapchains        = &info.swap_chain;
        present.pImageIndices      = &info.current_buffer;
        present.pWaitSemaphores    = NULL;
        present.waitSemaphoreCount = 0;
        present.pResults           = NULL;
    
        // Make sure command buffer is finished before presenting
        VkResult res;
        do
        {
            res = vkWaitForFences(info.device, 1, &drawFence, VK_TRUE, FENCE_TIMEOUT);
        } while (res == VK_TIMEOUT);
        vkResetFences(info.device, 1, &drawFence);
    
        ASSERT_EQ(VK_SUCCESS, res);
        res = vkQueuePresentKHR(info.present_queue, &present);
        ASSERT_EQ(VK_SUCCESS, res);
    }
    
    // 100 separate primary cmd buffers, each with 1 Draw
    void PrimaryCommandBufferBenchmarkHundredIndividual(sample_info &info,
                                                        VkClearValue *clear_values,
                                                        VkFence drawFence,
                                                        VkSemaphore imageAcquiredSemaphore,
                                                        int numBuffers)
    {
        VkResult res;
    
        VkRenderPassBeginInfo rpBegin;
        rpBegin.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
        rpBegin.pNext                    = NULL;
        rpBegin.renderPass               = info.render_pass;
        rpBegin.framebuffer              = info.framebuffers[info.current_buffer];
        rpBegin.renderArea.offset.x      = 0;
        rpBegin.renderArea.offset.y      = 0;
        rpBegin.renderArea.extent.width  = info.width;
        rpBegin.renderArea.extent.height = info.height;
        rpBegin.clearValueCount          = 2;
        rpBegin.pClearValues             = clear_values;
    
        VkCommandBufferBeginInfo cmdBufferInfo = {};
        cmdBufferInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
        cmdBufferInfo.pNext                    = NULL;
        cmdBufferInfo.flags                    = 0;
        cmdBufferInfo.pInheritanceInfo         = NULL;
    
        for (int x = 0; x < numBuffers; x++)
        {
            vkBeginCommandBuffer(info.cmds[x], &cmdBufferInfo);
            vkCmdBeginRenderPass(info.cmds[x], &rpBegin, VK_SUBPASS_CONTENTS_INLINE);
            vkCmdBindPipeline(info.cmds[x], VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline);
            vkCmdBindDescriptorSets(info.cmds[x], VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline_layout,
                                    0, NUM_DESCRIPTOR_SETS, info.desc_set.data(), 0, NULL);
    
            const VkDeviceSize offsets[1] = {0};
            vkCmdBindVertexBuffers(info.cmds[x], 0, 1, &info.vertex_buffer.buf, offsets);
    
            init_viewports_array(info, x);
            init_scissors_array(info, x);
    
            vkCmdDraw(info.cmds[x], 0, 1, 0, 0);
            vkCmdEndRenderPass(info.cmds[x]);
            res = vkEndCommandBuffer(info.cmds[x]);
            ASSERT_EQ(VK_SUCCESS, res);
        }
    
        VkPipelineStageFlags pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
        VkSubmitInfo submitInfo[1]            = {};
        submitInfo[0].pNext                   = NULL;
        submitInfo[0].sType                   = VK_STRUCTURE_TYPE_SUBMIT_INFO;
        submitInfo[0].waitSemaphoreCount      = 1;
        submitInfo[0].pWaitSemaphores         = &imageAcquiredSemaphore;
        submitInfo[0].pWaitDstStageMask       = &pipe_stage_flags;
        submitInfo[0].commandBufferCount      = numBuffers;
        submitInfo[0].pCommandBuffers         = info.cmds.data();
        submitInfo[0].signalSemaphoreCount    = 0;
        submitInfo[0].pSignalSemaphores       = NULL;
    
        // Queue the command buffer for execution
        res = vkQueueSubmit(info.graphics_queue, 1, submitInfo, drawFence);
        ASSERT_EQ(VK_SUCCESS, res);
    
        Present(info, drawFence);
    }
    
    // 100 of the same Draw cmds in the same primary cmd buffer
    void PrimaryCommandBufferBenchmarkOneWithOneHundred(sample_info &info,
                                                        VkClearValue *clear_values,
                                                        VkFence drawFence,
                                                        VkSemaphore imageAcquiredSemaphore,
                                                        int numBuffers)
    {
        VkResult res;
    
        VkRenderPassBeginInfo rpBegin;
        rpBegin.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
        rpBegin.pNext                    = NULL;
        rpBegin.renderPass               = info.render_pass;
        rpBegin.framebuffer              = info.framebuffers[info.current_buffer];
        rpBegin.renderArea.offset.x      = 0;
        rpBegin.renderArea.offset.y      = 0;
        rpBegin.renderArea.extent.width  = info.width;
        rpBegin.renderArea.extent.height = info.height;
        rpBegin.clearValueCount          = 2;
        rpBegin.pClearValues             = clear_values;
    
        VkCommandBufferBeginInfo cmdBufferInfo = {};
        cmdBufferInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
        cmdBufferInfo.pNext                    = NULL;
        cmdBufferInfo.flags                    = 0;
        cmdBufferInfo.pInheritanceInfo         = NULL;
    
        vkBeginCommandBuffer(info.cmd, &cmdBufferInfo);
        for (int x = 0; x < numBuffers; x++)
        {
            vkCmdBeginRenderPass(info.cmd, &rpBegin, VK_SUBPASS_CONTENTS_INLINE);
            vkCmdBindPipeline(info.cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline);
            vkCmdBindDescriptorSets(info.cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline_layout, 0,
                                    NUM_DESCRIPTOR_SETS, info.desc_set.data(), 0, NULL);
    
            const VkDeviceSize offsets[1] = {0};
            vkCmdBindVertexBuffers(info.cmd, 0, 1, &info.vertex_buffer.buf, offsets);
    
            init_viewports(info);
            init_scissors(info);
    
            vkCmdDraw(info.cmd, 0, 1, 0, 0);
            vkCmdEndRenderPass(info.cmd);
        }
        res = vkEndCommandBuffer(info.cmd);
        ASSERT_EQ(VK_SUCCESS, res);
    
        const VkCommandBuffer cmd_bufs[]      = {info.cmd};
        VkPipelineStageFlags pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
        VkSubmitInfo submitInfo[1]            = {};
        submitInfo[0].pNext                   = NULL;
        submitInfo[0].sType                   = VK_STRUCTURE_TYPE_SUBMIT_INFO;
        submitInfo[0].waitSemaphoreCount      = 1;
        submitInfo[0].pWaitSemaphores         = &imageAcquiredSemaphore;
        submitInfo[0].pWaitDstStageMask       = &pipe_stage_flags;
        submitInfo[0].commandBufferCount      = 1;
        submitInfo[0].pCommandBuffers         = cmd_bufs;
        submitInfo[0].signalSemaphoreCount    = 0;
        submitInfo[0].pSignalSemaphores       = NULL;
    
        // Queue the command buffer for execution
        res = vkQueueSubmit(info.graphics_queue, 1, submitInfo, drawFence);
        ASSERT_EQ(VK_SUCCESS, res);
    
        Present(info, drawFence);
    }
    
    // 100 separate secondary cmd buffers, each with 1 Draw
    void SecondaryCommandBufferBenchmark(sample_info &info,
                                         VkClearValue *clear_values,
                                         VkFence drawFence,
                                         VkSemaphore imageAcquiredSemaphore,
                                         int numBuffers)
    {
        VkResult res;
    
        // Record Secondary Command Buffer
        VkCommandBufferInheritanceInfo inheritInfo = {};
        inheritInfo.sType                          = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
        inheritInfo.pNext                          = NULL;
        inheritInfo.renderPass                     = info.render_pass;
        inheritInfo.subpass                        = 0;
        inheritInfo.framebuffer                    = info.framebuffers[info.current_buffer];
        inheritInfo.occlusionQueryEnable           = false;
        inheritInfo.queryFlags                     = 0;
        inheritInfo.pipelineStatistics             = 0;
    
        VkCommandBufferBeginInfo secondaryCommandBufferInfo = {};
        secondaryCommandBufferInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
        secondaryCommandBufferInfo.pNext = NULL;
        secondaryCommandBufferInfo.flags = VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT |
                                           VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT;
        secondaryCommandBufferInfo.pInheritanceInfo = &inheritInfo;
    
        for (int x = 0; x < numBuffers; x++)
        {
            vkBeginCommandBuffer(info.cmd2s[x], &secondaryCommandBufferInfo);
            vkCmdBindPipeline(info.cmd2s[x], VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline);
            vkCmdBindDescriptorSets(info.cmd2s[x], VK_PIPELINE_BIND_POINT_GRAPHICS,
                                    info.pipeline_layout, 0, NUM_DESCRIPTOR_SETS, info.desc_set.data(),
                                    0, NULL);
            const VkDeviceSize offsets[1] = {0};
            vkCmdBindVertexBuffers(info.cmd2s[x], 0, 1, &info.vertex_buffer.buf, offsets);
            init_viewports2_array(info, x);
            init_scissors2_array(info, x);
            vkCmdDraw(info.cmd2s[x], 0, 1, 0, 0);
            vkEndCommandBuffer(info.cmd2s[x]);
        }
        // Record Secondary Command Buffer End
    
        // Record Primary Command Buffer Begin
        VkRenderPassBeginInfo rpBegin;
        rpBegin.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
        rpBegin.pNext                    = NULL;
        rpBegin.renderPass               = info.render_pass;
        rpBegin.framebuffer              = info.framebuffers[info.current_buffer];
        rpBegin.renderArea.offset.x      = 0;
        rpBegin.renderArea.offset.y      = 0;
        rpBegin.renderArea.extent.width  = info.width;
        rpBegin.renderArea.extent.height = info.height;
        rpBegin.clearValueCount          = 2;
        rpBegin.pClearValues             = clear_values;
    
        VkCommandBufferBeginInfo primaryCommandBufferInfo = {};
        primaryCommandBufferInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
        primaryCommandBufferInfo.pNext                    = NULL;
        primaryCommandBufferInfo.flags                    = 0;
        primaryCommandBufferInfo.pInheritanceInfo         = NULL;
    
        vkBeginCommandBuffer(info.cmd, &primaryCommandBufferInfo);
        for (int x = 0; x < numBuffers; x++)
        {
            vkCmdBeginRenderPass(info.cmd, &rpBegin, VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS);
            vkCmdExecuteCommands(info.cmd, 1, &info.cmd2s[x]);
            vkCmdEndRenderPass(info.cmd);
        }
        vkEndCommandBuffer(info.cmd);
        // Record Primary Command Buffer End
    
        const VkCommandBuffer cmd_bufs[]      = {info.cmd};
        VkPipelineStageFlags pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
        VkSubmitInfo submitInfo[1]            = {};
        submitInfo[0].pNext                   = NULL;
        submitInfo[0].sType                   = VK_STRUCTURE_TYPE_SUBMIT_INFO;
        submitInfo[0].waitSemaphoreCount      = 1;
        submitInfo[0].pWaitSemaphores         = &imageAcquiredSemaphore;
        submitInfo[0].pWaitDstStageMask       = &pipe_stage_flags;
        submitInfo[0].commandBufferCount      = 1;
        submitInfo[0].pCommandBuffers         = cmd_bufs;
        submitInfo[0].signalSemaphoreCount    = 0;
        submitInfo[0].pSignalSemaphores       = NULL;
    
        // Queue the command buffer for execution
        res = vkQueueSubmit(info.graphics_queue, 1, submitInfo, drawFence);
        ASSERT_EQ(VK_SUCCESS, res);
    
        Present(info, drawFence);
    }
    
    // Details on the following functions that stress various cmd buffer reset methods.
    // All of these functions wrap the SecondaryCommandBufferBenchmark() test above,
    // adding additional overhead with various reset methods.
    // -CommandPoolDestroyBenchmark: Reset command buffers by destroying and re-creating
    //   command buffer pool.
    // -CommandPoolHardResetBenchmark: Reset the command pool w/ the RELEASE_RESOURCES
    //   bit set.
    // -CommandPoolSoftResetBenchmark: Reset to command pool w/o the RELEASE_RESOURCES
    //   bit set.
    // -CommandBufferExplicitHardResetBenchmark: Reset each individual command buffer
    //   w/ the RELEASE_RESOURCES bit set.
    // -CommandBufferExplicitSoftResetBenchmark: Reset each individual command buffer
    //   w/o the RELEASE_RESOURCES bit set.
    // -CommandBufferImplicitResetBenchmark: Reset each individual command buffer
    //   implicitly by calling "Begin" on previously used cmd buffer.
    
    void CommandPoolDestroyBenchmark(sample_info &info,
                                     VkClearValue *clear_values,
                                     VkFence drawFence,
                                     VkSemaphore imageAcquiredSemaphore,
                                     int numBuffers)
    {
        // Save setup cmd buffer data to be restored
        auto saved_cmd_pool = info.cmd_pool;
        auto saved_cb       = info.cmd;
        auto saved_cb2s     = info.cmd2s;
        // Now re-allocate & destroy cmd buffers to stress those calls
        init_command_pool(info, 0);
        init_command_buffer2_array(info, numBuffers);
    
        SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                        numBuffers);
    
        destroy_command_pool(info);
    
        // Restore original cmd buffer data for cleanup
        info.cmd_pool = saved_cmd_pool;
        info.cmd      = saved_cb;
        info.cmd2s    = saved_cb2s;
    }
    
    void CommandPoolHardResetBenchmark(sample_info &info,
                                       VkClearValue *clear_values,
                                       VkFence drawFence,
                                       VkSemaphore imageAcquiredSemaphore,
                                       int numBuffers)
    {
        SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                        numBuffers);
        reset_command_pool(info, VK_COMMAND_POOL_RESET_RELEASE_RESOURCES_BIT);
    }
    
    void CommandPoolSoftResetBenchmark(sample_info &info,
                                       VkClearValue *clear_values,
                                       VkFence drawFence,
                                       VkSemaphore imageAcquiredSemaphore,
                                       int numBuffers)
    {
        SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                        numBuffers);
        reset_command_pool(info, 0);
    }
    
    void CommandBufferExplicitHardResetBenchmark(sample_info &info,
                                                 VkClearValue *clear_values,
                                                 VkFence drawFence,
                                                 VkSemaphore imageAcquiredSemaphore,
                                                 int numBuffers)
    {
        SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                        numBuffers);
        // Explicitly resetting cmd buffers
        reset_command_buffer2_array(info, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
    }
    
    void CommandBufferExplicitSoftResetBenchmark(sample_info &info,
                                                 VkClearValue *clear_values,
                                                 VkFence drawFence,
                                                 VkSemaphore imageAcquiredSemaphore,
                                                 int numBuffers)
    {
        SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                        numBuffers);
        // Explicitly resetting cmd buffers w/ soft reset (don't release resources)
        reset_command_buffer2_array(info, 0);
    }
    
    void CommandBufferImplicitResetBenchmark(sample_info &info,
                                             VkClearValue *clear_values,
                                             VkFence drawFence,
                                             VkSemaphore imageAcquiredSemaphore,
                                             int numBuffers)
    {
        // Repeated call SCBBenchmark & BeginCmdBuffer calls will implicitly reset each cmd buffer
        SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                        numBuffers);
    }
    
    CommandBufferTestParams PrimaryCBHundredIndividualParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = PrimaryCommandBufferBenchmarkHundredIndividual;
        params.story            = "_PrimaryCB_Submit_100_With_1_Draw";
        return params;
    }
    
    CommandBufferTestParams PrimaryCBOneWithOneHundredParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = PrimaryCommandBufferBenchmarkOneWithOneHundred;
        params.story            = "_PrimaryCB_Submit_1_With_100_Draw";
        return params;
    }
    
    CommandBufferTestParams SecondaryCBParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = SecondaryCommandBufferBenchmark;
        params.story            = "_SecondaryCB_Submit_1_With_100_Draw_In_Individual_Secondary";
        return params;
    }
    
    CommandBufferTestParams CommandPoolDestroyParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = CommandPoolDestroyBenchmark;
        params.story            = "_Reset_CBs_With_Destroy_Command_Pool";
        return params;
    }
    
    CommandBufferTestParams CommandPoolHardResetParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = CommandPoolHardResetBenchmark;
        params.story            = "_Reset_CBs_With_Hard_Reset_Command_Pool";
        return params;
    }
    
    CommandBufferTestParams CommandPoolSoftResetParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = CommandPoolSoftResetBenchmark;
        params.story            = "_Reset_CBs_With_Soft_Reset_Command_Pool";
        return params;
    }
    
    CommandBufferTestParams CommandBufferExplicitHardResetParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = CommandBufferExplicitHardResetBenchmark;
        params.story            = "_Reset_CBs_With_Explicit_Hard_Reset_Command_Buffers";
        return params;
    }
    
    CommandBufferTestParams CommandBufferExplicitSoftResetParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = CommandBufferExplicitSoftResetBenchmark;
        params.story            = "_Reset_CBs_With_Explicit_Soft_Reset_Command_Buffers";
        return params;
    }
    
    CommandBufferTestParams CommandBufferImplicitResetParams()
    {
        CommandBufferTestParams params;
        params.CBImplementation = CommandBufferImplicitResetBenchmark;
        params.story            = "_Reset_CBs_With_Implicit_Reset_Command_Buffers";
        return params;
    }
    
    TEST_P(VulkanCommandBufferPerfTest, Run)
    {
        run();
    }
    
    INSTANTIATE_TEST_SUITE_P(,
                             VulkanCommandBufferPerfTest,
                             ::testing::Values(PrimaryCBHundredIndividualParams(),
                                               PrimaryCBOneWithOneHundredParams(),
                                               SecondaryCBParams(),
                                               CommandPoolDestroyParams(),
                                               CommandPoolHardResetParams(),
                                               CommandPoolSoftResetParams(),
                                               CommandBufferExplicitHardResetParams(),
                                               CommandBufferExplicitSoftResetParams(),
                                               CommandBufferImplicitResetParams()));