kc3-lang/angle/src/tests/perf_tests/VulkanCommandBufferPerf.cpp

Branch : - branch - main - tag -

• Show log

  Commit

• Author : Tobin Ehlis
  Date : 2019-02-06 12:24:19
  Hash : 3571ad49
  Message : Vulkan:Adding Cmd Buffer Reset tests Resetting cmd buffers in various ways to understand VK driver perf on various devices. Also includes some changes to get the chromium APK wrapper for angle_white_box_perftests compiling. Bug: angleproject:3137 Change-Id: I56bd163cdd6605ce09effe509a9c7ac5e0008e7a Reviewed-on: https://chromium-review.googlesource.com/c/1456482 Commit-Queue: Tobin Ehlis <tobine@google.com> Reviewed-by: Jamie Madill <jmadill@chromium.org>

• src/tests/perf_tests/VulkanCommandBufferPerf.cpp

• // Copyright 2018 The ANGLE Project Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  //
  // VulkanCommandBufferPerf:
  //   Performance benchmark for Vulkan Primary/Secondary Command Buffer implementations.
  
  #include "ANGLEPerfTest.h"
  #include "common/platform.h"
  #include "test_utils/third_party/vulkan_command_buffer_utils.h"
  
  #if defined(ANDROID)
  #    define NUM_CMD_BUFFERS 1000
  #    define NUM_FRAMES 10
  #else
  #    define NUM_CMD_BUFFERS 1000
  #    define NUM_FRAMES 100
  #endif
  
  constexpr char kVertShaderText[] = R"(
  #version 400
  #extension GL_ARB_separate_shader_objects : enable
  #extension GL_ARB_shading_language_420pack : enable
  layout (std140, binding = 0) uniform bufferVals {
      mat4 mvp;
  } myBufferVals;
  layout (location = 0) in vec4 pos;
  layout (location = 1) in vec4 inColor;
  layout (location = 0) out vec4 outColor;
  void main() {
     outColor = inColor;
     gl_Position = myBufferVals.mvp * pos;
  })";
  
  constexpr char kFragShaderText[] = R"(
  #version 400
  #extension GL_ARB_separate_shader_objects : enable
  #extension GL_ARB_shading_language_420pack : enable
  layout (location = 0) in vec4 color;
  layout (location = 0) out vec4 outColor;
  void main() {
     outColor = color;
  })";
  
  using CommandBufferImpl = void (*)(sample_info &info,
                                     VkClearValue *clear_values,
                                     VkFence drawFence,
                                     VkSemaphore imageAcquiredSemaphore,
                                     int numBuffers);
  
  struct CommandBufferTestParams
  {
      CommandBufferImpl CBImplementation;
      std::string suffix;
      int frames  = NUM_FRAMES;
      int buffers = NUM_CMD_BUFFERS;
  };
  
  class VulkanCommandBufferPerfTest : public ANGLEPerfTest,
                                      public ::testing::WithParamInterface<CommandBufferTestParams>
  {
    public:
      VulkanCommandBufferPerfTest();
  
      void SetUp() override;
      void TearDown() override;
      void step() override;
  
    private:
      VkClearValue mClearValues[2]        = {};
      VkSemaphore mImageAcquiredSemaphore = VK_NULL_HANDLE;
      VkFence mDrawFence                  = VK_NULL_HANDLE;
  
      VkResult res             = VK_NOT_READY;
      const bool mDepthPresent = true;
      struct sample_info mInfo = {};
      std::string mSampleTitle;
      CommandBufferImpl mCBImplementation = nullptr;
      int mFrames                         = 0;
      int mBuffers                        = 0;
  };
  
  VulkanCommandBufferPerfTest::VulkanCommandBufferPerfTest()
      : ANGLEPerfTest("VulkanCommandBufferPerfTest", GetParam().suffix, GetParam().frames)
  {
      mInfo             = {};
      mSampleTitle      = "Draw Textured Cube";
      mCBImplementation = GetParam().CBImplementation;
      mFrames           = GetParam().frames;
      mBuffers          = GetParam().buffers;
  
  // This test appears to be flaky on multiple platforms.
  #if !defined(ANGLE_PLATFORM_ANDROID)
      // mSkipTest = true;
  #endif  // !defined(ANGLE_PLATFORM_ANDROID)
  }
  
  void VulkanCommandBufferPerfTest::SetUp()
  {
      if (mSkipTest)
      {
          return;
      }
  
      init_global_layer_properties(mInfo);
      init_instance_extension_names(mInfo);
      init_device_extension_names(mInfo);
      init_instance(mInfo, mSampleTitle.c_str());
      init_enumerate_device(mInfo);
      init_window_size(mInfo, 500, 500);
      init_connection(mInfo);
      init_window(mInfo);
      init_swapchain_extension(mInfo);
      init_device(mInfo);
  
      init_command_pool(mInfo, VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT);
      init_command_buffer(mInfo);                   // Primary command buffer to hold secondaries
      init_command_buffer_array(mInfo, mBuffers);   // Array of primary command buffers
      init_command_buffer2_array(mInfo, mBuffers);  // Array containing all secondary buffers
      init_device_queue(mInfo);
      init_swap_chain(mInfo);
      init_depth_buffer(mInfo);
      init_uniform_buffer(mInfo);
      init_descriptor_and_pipeline_layouts(mInfo, false);
      init_renderpass(mInfo, mDepthPresent);
      init_shaders(mInfo, kVertShaderText, kFragShaderText);
      init_framebuffers(mInfo, mDepthPresent);
      init_vertex_buffer(mInfo, g_vb_solid_face_colors_Data, sizeof(g_vb_solid_face_colors_Data),
                         sizeof(g_vb_solid_face_colors_Data[0]), false);
      init_descriptor_pool(mInfo, false);
      init_descriptor_set(mInfo);
      init_pipeline_cache(mInfo);
      init_pipeline(mInfo, mDepthPresent);
  
      mClearValues[0].color.float32[0]     = 0.2f;
      mClearValues[0].color.float32[1]     = 0.2f;
      mClearValues[0].color.float32[2]     = 0.2f;
      mClearValues[0].color.float32[3]     = 0.2f;
      mClearValues[1].depthStencil.depth   = 1.0f;
      mClearValues[1].depthStencil.stencil = 0;
  
      VkSemaphoreCreateInfo imageAcquiredSemaphoreCreateInfo;
      imageAcquiredSemaphoreCreateInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
      imageAcquiredSemaphoreCreateInfo.pNext = NULL;
      imageAcquiredSemaphoreCreateInfo.flags = 0;
      res = vkCreateSemaphore(mInfo.device, &imageAcquiredSemaphoreCreateInfo, NULL,
                              &mImageAcquiredSemaphore);
      ASSERT_EQ(VK_SUCCESS, res);
  
      VkFenceCreateInfo fenceInfo;
      fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
      fenceInfo.pNext = NULL;
      fenceInfo.flags = 0;
      res             = vkCreateFence(mInfo.device, &fenceInfo, NULL, &mDrawFence);
      ASSERT_EQ(VK_SUCCESS, res);
  }
  
  void VulkanCommandBufferPerfTest::step()
  {
      for (int x = 0; x < mFrames; x++)
      {
          mInfo.current_buffer = x % mInfo.swapchainImageCount;
  
          // Get the index of the next available swapchain image:
          res = vkAcquireNextImageKHR(mInfo.device, mInfo.swap_chain, UINT64_MAX,
                                      mImageAcquiredSemaphore, VK_NULL_HANDLE, &mInfo.current_buffer);
          // Deal with the VK_SUBOPTIMAL_KHR and VK_ERROR_OUT_OF_DATE_KHR
          // return codes
          ASSERT_EQ(VK_SUCCESS, res);
          mCBImplementation(mInfo, mClearValues, mDrawFence, mImageAcquiredSemaphore, mBuffers);
      }
  }
  
  void VulkanCommandBufferPerfTest::TearDown()
  {
      if (mSkipTest)
      {
          return;
      }
  
      vkDestroySemaphore(mInfo.device, mImageAcquiredSemaphore, NULL);
      vkDestroyFence(mInfo.device, mDrawFence, NULL);
      destroy_pipeline(mInfo);
      destroy_pipeline_cache(mInfo);
      destroy_descriptor_pool(mInfo);
      destroy_vertex_buffer(mInfo);
      destroy_framebuffers(mInfo);
      destroy_shaders(mInfo);
      destroy_renderpass(mInfo);
      destroy_descriptor_and_pipeline_layouts(mInfo);
      destroy_uniform_buffer(mInfo);
      destroy_depth_buffer(mInfo);
      destroy_swap_chain(mInfo);
      destroy_command_buffer2_array(mInfo, mBuffers);
      destroy_command_buffer_array(mInfo, mBuffers);
      destroy_command_buffer(mInfo);
      destroy_command_pool(mInfo);
      destroy_device(mInfo);
      destroy_window(mInfo);
      destroy_instance(mInfo);
      ANGLEPerfTest::TearDown();
  }
  
  // Common code to present image used by all tests
  void Present(sample_info &info, VkFence drawFence)
  {
      // Now present the image in the window
  
      VkPresentInfoKHR present;
      present.sType              = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
      present.pNext              = NULL;
      present.swapchainCount     = 1;
      present.pSwapchains        = &info.swap_chain;
      present.pImageIndices      = &info.current_buffer;
      present.pWaitSemaphores    = NULL;
      present.waitSemaphoreCount = 0;
      present.pResults           = NULL;
  
      // Make sure command buffer is finished before presenting
      VkResult res;
      do
      {
          res = vkWaitForFences(info.device, 1, &drawFence, VK_TRUE, FENCE_TIMEOUT);
      } while (res == VK_TIMEOUT);
      vkResetFences(info.device, 1, &drawFence);
  
      ASSERT_EQ(VK_SUCCESS, res);
      res = vkQueuePresentKHR(info.present_queue, &present);
      ASSERT_EQ(VK_SUCCESS, res);
  }
  
  void PrimaryCommandBufferBenchmarkHundredIndividual(sample_info &info,
                                                      VkClearValue *clear_values,
                                                      VkFence drawFence,
                                                      VkSemaphore imageAcquiredSemaphore,
                                                      int numBuffers)
  {
      VkResult res;
  
      VkRenderPassBeginInfo rpBegin;
      rpBegin.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
      rpBegin.pNext                    = NULL;
      rpBegin.renderPass               = info.render_pass;
      rpBegin.framebuffer              = info.framebuffers[info.current_buffer];
      rpBegin.renderArea.offset.x      = 0;
      rpBegin.renderArea.offset.y      = 0;
      rpBegin.renderArea.extent.width  = info.width;
      rpBegin.renderArea.extent.height = info.height;
      rpBegin.clearValueCount          = 2;
      rpBegin.pClearValues             = clear_values;
  
      VkCommandBufferBeginInfo cmdBufferInfo = {};
      cmdBufferInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
      cmdBufferInfo.pNext                    = NULL;
      cmdBufferInfo.flags                    = 0;
      cmdBufferInfo.pInheritanceInfo         = NULL;
  
      for (int x = 0; x < numBuffers; x++)
      {
          vkBeginCommandBuffer(info.cmds[x], &cmdBufferInfo);
          vkCmdBeginRenderPass(info.cmds[x], &rpBegin, VK_SUBPASS_CONTENTS_INLINE);
          vkCmdBindPipeline(info.cmds[x], VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline);
          vkCmdBindDescriptorSets(info.cmds[x], VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline_layout,
                                  0, NUM_DESCRIPTOR_SETS, info.desc_set.data(), 0, NULL);
  
          const VkDeviceSize offsets[1] = {0};
          vkCmdBindVertexBuffers(info.cmds[x], 0, 1, &info.vertex_buffer.buf, offsets);
  
          init_viewports_array(info, x);
          init_scissors_array(info, x);
  
          vkCmdDraw(info.cmds[x], 0, 1, 0, 0);
          vkCmdEndRenderPass(info.cmds[x]);
          res = vkEndCommandBuffer(info.cmds[x]);
          ASSERT_EQ(VK_SUCCESS, res);
      }
  
      VkPipelineStageFlags pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
      VkSubmitInfo submitInfo[1]            = {};
      submitInfo[0].pNext                   = NULL;
      submitInfo[0].sType                   = VK_STRUCTURE_TYPE_SUBMIT_INFO;
      submitInfo[0].waitSemaphoreCount      = 1;
      submitInfo[0].pWaitSemaphores         = &imageAcquiredSemaphore;
      submitInfo[0].pWaitDstStageMask       = &pipe_stage_flags;
      submitInfo[0].commandBufferCount      = numBuffers;
      submitInfo[0].pCommandBuffers         = info.cmds.data();
      submitInfo[0].signalSemaphoreCount    = 0;
      submitInfo[0].pSignalSemaphores       = NULL;
  
      // Queue the command buffer for execution
      res = vkQueueSubmit(info.graphics_queue, 1, submitInfo, drawFence);
      ASSERT_EQ(VK_SUCCESS, res);
  
      Present(info, drawFence);
  }
  
  void PrimaryCommandBufferBenchmarkOneWithOneHundred(sample_info &info,
                                                      VkClearValue *clear_values,
                                                      VkFence drawFence,
                                                      VkSemaphore imageAcquiredSemaphore,
                                                      int numBuffers)
  {
      VkResult res;
  
      VkRenderPassBeginInfo rpBegin;
      rpBegin.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
      rpBegin.pNext                    = NULL;
      rpBegin.renderPass               = info.render_pass;
      rpBegin.framebuffer              = info.framebuffers[info.current_buffer];
      rpBegin.renderArea.offset.x      = 0;
      rpBegin.renderArea.offset.y      = 0;
      rpBegin.renderArea.extent.width  = info.width;
      rpBegin.renderArea.extent.height = info.height;
      rpBegin.clearValueCount          = 2;
      rpBegin.pClearValues             = clear_values;
  
      VkCommandBufferBeginInfo cmdBufferInfo = {};
      cmdBufferInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
      cmdBufferInfo.pNext                    = NULL;
      cmdBufferInfo.flags                    = 0;
      cmdBufferInfo.pInheritanceInfo         = NULL;
  
      vkBeginCommandBuffer(info.cmd, &cmdBufferInfo);
      for (int x = 0; x < numBuffers; x++)
      {
          vkCmdBeginRenderPass(info.cmd, &rpBegin, VK_SUBPASS_CONTENTS_INLINE);
          vkCmdBindPipeline(info.cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline);
          vkCmdBindDescriptorSets(info.cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline_layout, 0,
                                  NUM_DESCRIPTOR_SETS, info.desc_set.data(), 0, NULL);
  
          const VkDeviceSize offsets[1] = {0};
          vkCmdBindVertexBuffers(info.cmd, 0, 1, &info.vertex_buffer.buf, offsets);
  
          init_viewports(info);
          init_scissors(info);
  
          vkCmdDraw(info.cmd, 0, 1, 0, 0);
          vkCmdEndRenderPass(info.cmd);
      }
      res = vkEndCommandBuffer(info.cmd);
      ASSERT_EQ(VK_SUCCESS, res);
  
      const VkCommandBuffer cmd_bufs[]      = {info.cmd};
      VkPipelineStageFlags pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
      VkSubmitInfo submitInfo[1]            = {};
      submitInfo[0].pNext                   = NULL;
      submitInfo[0].sType                   = VK_STRUCTURE_TYPE_SUBMIT_INFO;
      submitInfo[0].waitSemaphoreCount      = 1;
      submitInfo[0].pWaitSemaphores         = &imageAcquiredSemaphore;
      submitInfo[0].pWaitDstStageMask       = &pipe_stage_flags;
      submitInfo[0].commandBufferCount      = 1;
      submitInfo[0].pCommandBuffers         = cmd_bufs;
      submitInfo[0].signalSemaphoreCount    = 0;
      submitInfo[0].pSignalSemaphores       = NULL;
  
      // Queue the command buffer for execution
      res = vkQueueSubmit(info.graphics_queue, 1, submitInfo, drawFence);
      ASSERT_EQ(VK_SUCCESS, res);
  
      Present(info, drawFence);
  }
  
  void SecondaryCommandBufferBenchmark(sample_info &info,
                                       VkClearValue *clear_values,
                                       VkFence drawFence,
                                       VkSemaphore imageAcquiredSemaphore,
                                       int numBuffers)
  {
      VkResult res;
  
      // Record Secondary Command Buffer
      VkCommandBufferInheritanceInfo inheritInfo = {};
      inheritInfo.sType                          = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
      inheritInfo.pNext                          = NULL;
      inheritInfo.renderPass                     = info.render_pass;
      inheritInfo.subpass                        = 0;
      inheritInfo.framebuffer                    = info.framebuffers[info.current_buffer];
      inheritInfo.occlusionQueryEnable           = false;
      inheritInfo.queryFlags                     = 0;
      inheritInfo.pipelineStatistics             = 0;
  
      VkCommandBufferBeginInfo secondaryCommandBufferInfo = {};
      secondaryCommandBufferInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
      secondaryCommandBufferInfo.pNext = NULL;
      secondaryCommandBufferInfo.flags = VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT |
                                         VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT;
      secondaryCommandBufferInfo.pInheritanceInfo = &inheritInfo;
  
      for (int x = 0; x < numBuffers; x++)
      {
          vkBeginCommandBuffer(info.cmd2s[x], &secondaryCommandBufferInfo);
          vkCmdBindPipeline(info.cmd2s[x], VK_PIPELINE_BIND_POINT_GRAPHICS, info.pipeline);
          vkCmdBindDescriptorSets(info.cmd2s[x], VK_PIPELINE_BIND_POINT_GRAPHICS,
                                  info.pipeline_layout, 0, NUM_DESCRIPTOR_SETS, info.desc_set.data(),
                                  0, NULL);
          const VkDeviceSize offsets[1] = {0};
          vkCmdBindVertexBuffers(info.cmd2s[x], 0, 1, &info.vertex_buffer.buf, offsets);
          init_viewports2_array(info, x);
          init_scissors2_array(info, x);
          vkCmdDraw(info.cmd2s[x], 0, 1, 0, 0);
          vkEndCommandBuffer(info.cmd2s[x]);
      }
      // Record Secondary Command Buffer End
  
      // Record Primary Command Buffer Begin
      VkRenderPassBeginInfo rpBegin;
      rpBegin.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
      rpBegin.pNext                    = NULL;
      rpBegin.renderPass               = info.render_pass;
      rpBegin.framebuffer              = info.framebuffers[info.current_buffer];
      rpBegin.renderArea.offset.x      = 0;
      rpBegin.renderArea.offset.y      = 0;
      rpBegin.renderArea.extent.width  = info.width;
      rpBegin.renderArea.extent.height = info.height;
      rpBegin.clearValueCount          = 2;
      rpBegin.pClearValues             = clear_values;
  
      VkCommandBufferBeginInfo primaryCommandBufferInfo = {};
      primaryCommandBufferInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
      primaryCommandBufferInfo.pNext                    = NULL;
      primaryCommandBufferInfo.flags                    = 0;
      primaryCommandBufferInfo.pInheritanceInfo         = NULL;
  
      vkBeginCommandBuffer(info.cmd, &primaryCommandBufferInfo);
      for (int x = 0; x < numBuffers; x++)
      {
          vkCmdBeginRenderPass(info.cmd, &rpBegin, VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS);
          vkCmdExecuteCommands(info.cmd, 1, &info.cmd2s[x]);
          vkCmdEndRenderPass(info.cmd);
      }
      vkEndCommandBuffer(info.cmd);
      // Record Primary Command Buffer End
  
      const VkCommandBuffer cmd_bufs[]      = {info.cmd};
      VkPipelineStageFlags pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
      VkSubmitInfo submitInfo[1]            = {};
      submitInfo[0].pNext                   = NULL;
      submitInfo[0].sType                   = VK_STRUCTURE_TYPE_SUBMIT_INFO;
      submitInfo[0].waitSemaphoreCount      = 1;
      submitInfo[0].pWaitSemaphores         = &imageAcquiredSemaphore;
      submitInfo[0].pWaitDstStageMask       = &pipe_stage_flags;
      submitInfo[0].commandBufferCount      = 1;
      submitInfo[0].pCommandBuffers         = cmd_bufs;
      submitInfo[0].signalSemaphoreCount    = 0;
      submitInfo[0].pSignalSemaphores       = NULL;
  
      // Queue the command buffer for execution
      res = vkQueueSubmit(info.graphics_queue, 1, submitInfo, drawFence);
      ASSERT_EQ(VK_SUCCESS, res);
  
      Present(info, drawFence);
  }
  
  void CommandPoolDestroyBenchmark(sample_info &info,
                                   VkClearValue *clear_values,
                                   VkFence drawFence,
                                   VkSemaphore imageAcquiredSemaphore,
                                   int numBuffers)
  {
      // Save setup cmd buffer data to be restored
      auto saved_cmd_pool = info.cmd_pool;
      auto saved_cb       = info.cmd;
      auto saved_cb2s     = info.cmd2s;
      // Now re-allocate & destroy cmd buffers to stress those calls
      init_command_pool(info, 0);
      init_command_buffer2_array(info, numBuffers);
  
      SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                      numBuffers);
  
      destroy_command_pool(info);
  
      // Restore original cmd buffer data for cleanup
      info.cmd_pool = saved_cmd_pool;
      info.cmd      = saved_cb;
      info.cmd2s    = saved_cb2s;
  }
  
  void CommandPoolHardResetBenchmark(sample_info &info,
                                     VkClearValue *clear_values,
                                     VkFence drawFence,
                                     VkSemaphore imageAcquiredSemaphore,
                                     int numBuffers)
  {
      SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                      numBuffers);
      reset_command_pool(info, VK_COMMAND_POOL_RESET_RELEASE_RESOURCES_BIT);
  }
  
  void CommandPoolSoftResetBenchmark(sample_info &info,
                                     VkClearValue *clear_values,
                                     VkFence drawFence,
                                     VkSemaphore imageAcquiredSemaphore,
                                     int numBuffers)
  {
      SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                      numBuffers);
      reset_command_pool(info, 0);
  }
  
  void CommandBufferExplicitHardResetBenchmark(sample_info &info,
                                               VkClearValue *clear_values,
                                               VkFence drawFence,
                                               VkSemaphore imageAcquiredSemaphore,
                                               int numBuffers)
  {
      SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                      numBuffers);
      // Explicitly resetting cmd buffers
      reset_command_buffer2_array(info, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
  }
  
  void CommandBufferExplicitSoftResetBenchmark(sample_info &info,
                                               VkClearValue *clear_values,
                                               VkFence drawFence,
                                               VkSemaphore imageAcquiredSemaphore,
                                               int numBuffers)
  {
      SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                      numBuffers);
      // Explicitly resetting cmd buffers w/ soft reset (don't release resources)
      reset_command_buffer2_array(info, 0);
  }
  
  void CommandBufferImplicitResetBenchmark(sample_info &info,
                                           VkClearValue *clear_values,
                                           VkFence drawFence,
                                           VkSemaphore imageAcquiredSemaphore,
                                           int numBuffers)
  {
      // Repeated call SCBBenchmark & BeginCmdBuffer calls will implicitly reset each cmd buffer
      SecondaryCommandBufferBenchmark(info, clear_values, drawFence, imageAcquiredSemaphore,
                                      numBuffers);
  }
  
  CommandBufferTestParams PrimaryCBHundredIndividualParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = PrimaryCommandBufferBenchmarkHundredIndividual;
      params.suffix           = "_PrimaryCB_Submit_100_With_1_Draw";
      return params;
  }
  
  CommandBufferTestParams PrimaryCBOneWithOneHundredParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = PrimaryCommandBufferBenchmarkOneWithOneHundred;
      params.suffix           = "_PrimaryCB_Submit_1_With_100_Draw";
      return params;
  }
  
  CommandBufferTestParams SecondaryCBParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = SecondaryCommandBufferBenchmark;
      params.suffix           = "_SecondaryCB_Submit_1_With_100_Draw_In_Individual_Secondary";
      return params;
  }
  
  CommandBufferTestParams CommandPoolDestroyParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = CommandPoolDestroyBenchmark;
      params.suffix           = "_Reset_CBs_With_Destroy_Command_Pool";
      return params;
  }
  
  CommandBufferTestParams CommandPoolHardResetParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = CommandPoolHardResetBenchmark;
      params.suffix           = "_Reset_CBs_With_Hard_Reset_Command_Pool";
      return params;
  }
  
  CommandBufferTestParams CommandPoolSoftResetParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = CommandPoolSoftResetBenchmark;
      params.suffix           = "_Reset_CBs_With_Soft_Reset_Command_Pool";
      return params;
  }
  
  CommandBufferTestParams CommandBufferExplicitHardResetParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = CommandBufferExplicitHardResetBenchmark;
      params.suffix           = "_Reset_CBs_With_Explicit_Hard_Reset_Command_Buffers";
      return params;
  }
  
  CommandBufferTestParams CommandBufferExplicitSoftResetParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = CommandBufferExplicitSoftResetBenchmark;
      params.suffix           = "_Reset_CBs_With_Explicit_Soft_Reset_Command_Buffers";
      return params;
  }
  
  CommandBufferTestParams CommandBufferImplicitResetParams()
  {
      CommandBufferTestParams params;
      params.CBImplementation = CommandBufferImplicitResetBenchmark;
      params.suffix           = "_Reset_CBs_With_Implicit_Reset_Command_Buffers";
      return params;
  }
  
  TEST_P(VulkanCommandBufferPerfTest, Run)
  {
      run();
  }
  
  INSTANTIATE_TEST_CASE_P(,
                          VulkanCommandBufferPerfTest,
                          ::testing::Values(PrimaryCBHundredIndividualParams(),
                                            PrimaryCBOneWithOneHundredParams(),
                                            SecondaryCBParams(),
                                            CommandPoolDestroyParams(),
                                            CommandPoolHardResetParams(),
                                            CommandPoolSoftResetParams(),
                                            CommandBufferExplicitHardResetParams(),
                                            CommandBufferExplicitSoftResetParams(),
                                            CommandBufferImplicitResetParams()));