Edit
IABSD.fr/xenocara/lib/mesa/src/amd/vulkan/radv_pipeline_binary.c
jsg
- lib/mesa/src/amd/vulkan/radv_pipeline_binary.c
-
/* * Copyright © 2024 Valve Corporation * * SPDX-License-Identifier: MIT */ #include "radv_pipeline_binary.h" #include "util/disk_cache.h" #include "util/macros.h" #include "util/mesa-blake3.h" #include "util/mesa-sha1.h" #include "util/u_atomic.h" #include "util/u_debug.h" #include "nir_serialize.h" #include "radv_debug.h" #include "radv_device.h" #include "radv_entrypoints.h" #include "radv_pipeline_cache.h" #include "radv_pipeline_graphics.h" #include "radv_pipeline_rt.h" #include "radv_shader.h" #include "vk_log.h" #include "vk_pipeline.h" #include "vk_util.h" static VkResult radv_get_pipeline_key(struct radv_device *device, const VkPipelineCreateInfoKHR *pPipelineCreateInfo, unsigned char *key) { VkResult result = VK_SUCCESS; switch (((VkBaseInStructure *)pPipelineCreateInfo->pNext)->sType) { case VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO: { const VkGraphicsPipelineCreateInfo *graphics_create_info = (VkGraphicsPipelineCreateInfo *)pPipelineCreateInfo->pNext; struct radv_graphics_pipeline_state gfx_state; result = radv_generate_graphics_pipeline_state(device, graphics_create_info, &gfx_state); if (result != VK_SUCCESS) return result; radv_graphics_pipeline_hash(device, &gfx_state, key); radv_graphics_pipeline_state_finish(device, &gfx_state); break; } case VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO: { const VkComputePipelineCreateInfo *compute_create_info = (VkComputePipelineCreateInfo *)pPipelineCreateInfo->pNext; radv_compute_pipeline_hash(device, compute_create_info, key); break; } case VK_STRUCTURE_TYPE_RAY_TRACING_PIPELINE_CREATE_INFO_KHR: { const VkRayTracingPipelineCreateInfoKHR *rt_create_info = (VkRayTracingPipelineCreateInfoKHR *)pPipelineCreateInfo->pNext; struct radv_ray_tracing_state_key rt_state; result = radv_generate_ray_tracing_state_key(device, rt_create_info, &rt_state); if (result != VK_SUCCESS) return result; radv_ray_tracing_pipeline_hash(device, rt_create_info, &rt_state, key); radv_ray_tracing_state_key_finish(&rt_state); break; } default: unreachable("unsupported pipeline create info struct"); } return result; } VKAPI_ATTR VkResult VKAPI_CALL radv_GetPipelineKeyKHR(VkDevice _device, const VkPipelineCreateInfoKHR *pPipelineCreateInfo, VkPipelineBinaryKeyKHR *pPipelineKey) { VK_FROM_HANDLE(radv_device, device, _device); const struct radv_physical_device *pdev = radv_device_physical(device); VkResult result; memset(pPipelineKey->key, 0, sizeof(pPipelineKey->key)); /* Return the global key that applies to all pipelines. */ if (!pPipelineCreateInfo) { struct mesa_blake3 ctx; static_assert(sizeof(blake3_hash) <= sizeof(pPipelineKey->key), "mismatch pipeline binary key size"); _mesa_blake3_init(&ctx); _mesa_blake3_update(&ctx, pdev->cache_uuid, sizeof(pdev->cache_uuid)); _mesa_blake3_update(&ctx, device->cache_hash, sizeof(device->cache_hash)); _mesa_blake3_final(&ctx, pPipelineKey->key); pPipelineKey->keySize = sizeof(blake3_hash); return VK_SUCCESS; } result = radv_get_pipeline_key(device, pPipelineCreateInfo, pPipelineKey->key); if (result != VK_SUCCESS) return result; pPipelineKey->keySize = SHA1_DIGEST_LENGTH; return VK_SUCCESS; } static VkResult radv_create_pipeline_binary(struct radv_device *device, const VkAllocationCallbacks *pAllocator, const blake3_hash key, const void *data, size_t data_size, struct radv_pipeline_binary **pipeline_binary_out) { struct radv_pipeline_binary *pipeline_binary; pipeline_binary = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*pipeline_binary), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (pipeline_binary == NULL) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); vk_object_base_init(&device->vk, &pipeline_binary->base, VK_OBJECT_TYPE_PIPELINE_BINARY_KHR); pipeline_binary->data = (void *)data; pipeline_binary->size = data_size; memcpy(pipeline_binary->key, key, BLAKE3_OUT_LEN); *pipeline_binary_out = pipeline_binary; return VK_SUCCESS; } static VkResult radv_create_pipeline_binary_from_data(struct radv_device *device, const VkAllocationCallbacks *pAllocator, const VkPipelineBinaryDataKHR *pData, const VkPipelineBinaryKeyKHR *pKey, struct util_dynarray *pipeline_binaries, uint32_t *num_binaries) { struct radv_pipeline_binary *pipeline_binary; VkResult result; void *data; if (!pipeline_binaries) { (*num_binaries)++; return VK_SUCCESS; } data = malloc(pData->dataSize); if (!data) return VK_ERROR_OUT_OF_HOST_MEMORY; memcpy(data, pData->pData, pData->dataSize); result = radv_create_pipeline_binary(device, pAllocator, pKey->key, data, pData->dataSize, &pipeline_binary); if (result != VK_SUCCESS) { free(data); return result; } util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary); return result; } VkResult radv_create_pipeline_binary_from_shader(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_shader *shader, struct util_dynarray *pipeline_binaries, uint32_t *num_binaries) { struct radv_pipeline_binary *pipeline_binary; struct blob blob; size_t data_size; VkResult result; void *data; if (!pipeline_binaries) { (*num_binaries)++; return VK_SUCCESS; } blob_init(&blob); radv_shader_serialize(shader, &blob); blob_finish_get_buffer(&blob, &data, &data_size); result = radv_create_pipeline_binary(device, pAllocator, shader->hash, data, data_size, &pipeline_binary); if (result != VK_SUCCESS) { free(data); return result; } util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary); return result; } VkResult radv_create_pipeline_binary_from_rt_shader(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_shader *shader, bool is_traversal_shader, const uint8_t stage_sha1[SHA1_DIGEST_LENGTH], const struct radv_ray_tracing_stage_info *rt_stage_info, uint32_t stack_size, struct vk_pipeline_cache_object *nir, struct util_dynarray *pipeline_binaries, uint32_t *num_binaries) { struct radv_pipeline_binary *pipeline_binary; struct mesa_blake3 ctx; struct blob blob; size_t data_size; blake3_hash key; VkResult result; void *data; if (!pipeline_binaries) { (*num_binaries)++; return VK_SUCCESS; } _mesa_blake3_init(&ctx); _mesa_blake3_update(&ctx, stage_sha1, sizeof(*stage_sha1)); _mesa_blake3_final(&ctx, key); struct radv_ray_tracing_binary_header header = { .is_traversal_shader = is_traversal_shader, .has_shader = !!shader, .has_nir = !!nir, .stack_size = stack_size, }; memcpy(header.stage_sha1, stage_sha1, sizeof(header.stage_sha1)); if (rt_stage_info) memcpy(&header.stage_info, rt_stage_info, sizeof(header.stage_info)); blob_init(&blob); blob_write_bytes(&blob, &header, sizeof(header)); if (header.has_shader) radv_shader_serialize(shader, &blob); if (header.has_nir) { struct vk_raw_data_cache_object *nir_object = container_of(nir, struct vk_raw_data_cache_object, base); blob_write_bytes(&blob, nir_object->data, nir_object->data_size); } blob_finish_get_buffer(&blob, &data, &data_size); result = radv_create_pipeline_binary(device, pAllocator, key, data, data_size, &pipeline_binary); if (result != VK_SUCCESS) { free(data); return result; } util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary); return result; } static VkResult radv_create_pipeline_binary_from_pipeline(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_pipeline *pipeline, struct util_dynarray *pipeline_binaries, uint32_t *num_binaries) { VkResult result = VK_SUCCESS; if (pipeline->type == RADV_PIPELINE_RAY_TRACING) { struct radv_ray_tracing_pipeline *rt_pipeline = radv_pipeline_to_ray_tracing(pipeline); for (uint32_t i = 0; i < rt_pipeline->non_imported_stage_count; i++) { struct radv_ray_tracing_stage *rt_stage = &rt_pipeline->stages[i]; result = radv_create_pipeline_binary_from_rt_shader(device, pAllocator, rt_stage->shader, false, rt_stage->sha1, &rt_stage->info, rt_stage->stack_size, rt_stage->nir, pipeline_binaries, num_binaries); if (result != VK_SUCCESS) return result; } struct radv_shader *traversal_shader = rt_pipeline->base.base.shaders[MESA_SHADER_INTERSECTION]; if (traversal_shader) { result = radv_create_pipeline_binary_from_rt_shader(device, pAllocator, traversal_shader, true, traversal_shader->hash, NULL, 0, NULL, pipeline_binaries, num_binaries); if (result != VK_SUCCESS) return result; } } else { for (uint32_t i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) { if (!pipeline->shaders[i]) continue; result = radv_create_pipeline_binary_from_shader(device, pAllocator, pipeline->shaders[i], pipeline_binaries, num_binaries); if (result != VK_SUCCESS) return result; } if (pipeline->gs_copy_shader) { result = radv_create_pipeline_binary_from_shader(device, pAllocator, pipeline->gs_copy_shader, pipeline_binaries, num_binaries); if (result != VK_SUCCESS) return result; } } return result; } static VkResult radv_create_pipeline_binary_from_cache(struct radv_device *device, const VkAllocationCallbacks *pAllocator, const VkPipelineCreateInfoKHR *pPipelineCreateInfo, struct util_dynarray *pipeline_binaries, uint32_t *num_binaries) { unsigned char key[SHA1_DIGEST_LENGTH]; bool found_in_internal_cache; VkResult result; assert(pPipelineCreateInfo); result = radv_get_pipeline_key(device, pPipelineCreateInfo, key); if (result != VK_SUCCESS) return result; result = radv_pipeline_cache_get_binaries(device, pAllocator, key, pipeline_binaries, num_binaries, &found_in_internal_cache); if (result != VK_SUCCESS) return result; return found_in_internal_cache ? VK_SUCCESS : VK_PIPELINE_BINARY_MISSING_KHR; } static VkResult radv_create_pipeline_binaries(struct radv_device *device, const VkPipelineBinaryCreateInfoKHR *pCreateInfo, const VkAllocationCallbacks *pAllocator, struct util_dynarray *pipeline_binaries, uint32_t *num_binaries) { VkResult result = VK_SUCCESS; if (pCreateInfo->pKeysAndDataInfo) { const VkPipelineBinaryKeysAndDataKHR *pKeysAndDataInfo = pCreateInfo->pKeysAndDataInfo; for (uint32_t i = 0; i < pKeysAndDataInfo->binaryCount; i++) { const VkPipelineBinaryDataKHR *pData = &pKeysAndDataInfo->pPipelineBinaryData[i]; const VkPipelineBinaryKeyKHR *pKey = &pKeysAndDataInfo->pPipelineBinaryKeys[i]; result = radv_create_pipeline_binary_from_data(device, pAllocator, pData, pKey, pipeline_binaries, num_binaries); if (result != VK_SUCCESS) return result; } } else if (pCreateInfo->pipeline) { VK_FROM_HANDLE(radv_pipeline, pipeline, pCreateInfo->pipeline); result = radv_create_pipeline_binary_from_pipeline(device, pAllocator, pipeline, pipeline_binaries, num_binaries); } else { result = radv_create_pipeline_binary_from_cache(device, pAllocator, pCreateInfo->pPipelineCreateInfo, pipeline_binaries, num_binaries); } return result; } static void radv_destroy_pipeline_binary(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_pipeline_binary *pipeline_binary) { if (!pipeline_binary) return; free(pipeline_binary->data); vk_object_base_finish(&pipeline_binary->base); vk_free2(&device->vk.alloc, pAllocator, pipeline_binary); } VKAPI_ATTR VkResult VKAPI_CALL radv_CreatePipelineBinariesKHR(VkDevice _device, const VkPipelineBinaryCreateInfoKHR *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkPipelineBinaryHandlesInfoKHR *pBinaries) { VK_FROM_HANDLE(radv_device, device, _device); struct util_dynarray pipeline_binaries; VkResult result; if (!pBinaries->pPipelineBinaries) { result = radv_create_pipeline_binaries(device, pCreateInfo, pAllocator, NULL, &pBinaries->pipelineBinaryCount); return result; } for (uint32_t i = 0; i < pBinaries->pipelineBinaryCount; i++) pBinaries->pPipelineBinaries[i] = VK_NULL_HANDLE; util_dynarray_init(&pipeline_binaries, NULL); /* Get all pipeline binaries from the pCreateInfo first to simplify the creation. */ result = radv_create_pipeline_binaries(device, pCreateInfo, pAllocator, &pipeline_binaries, NULL); if (result != VK_SUCCESS) { util_dynarray_foreach (&pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary) radv_destroy_pipeline_binary(device, pAllocator, *pipeline_binary); util_dynarray_fini(&pipeline_binaries); return result; } const uint32_t num_binaries = util_dynarray_num_elements(&pipeline_binaries, struct radv_pipeline_binary *); for (uint32_t i = 0; i < num_binaries; i++) { struct radv_pipeline_binary **pipeline_binary = util_dynarray_element(&pipeline_binaries, struct radv_pipeline_binary *, i); if (i < pBinaries->pipelineBinaryCount) { pBinaries->pPipelineBinaries[i] = radv_pipeline_binary_to_handle(*pipeline_binary); } else { /* Free the pipeline binary that couldn't be returned. */ radv_destroy_pipeline_binary(device, pAllocator, *pipeline_binary); } } result = pBinaries->pipelineBinaryCount < num_binaries ? VK_INCOMPLETE : result; pBinaries->pipelineBinaryCount = MIN2(num_binaries, pBinaries->pipelineBinaryCount); util_dynarray_fini(&pipeline_binaries); return result; } VKAPI_ATTR void VKAPI_CALL radv_DestroyPipelineBinaryKHR(VkDevice _device, VkPipelineBinaryKHR pipelineBinary, const VkAllocationCallbacks *pAllocator) { VK_FROM_HANDLE(radv_pipeline_binary, pipeline_binary, pipelineBinary); VK_FROM_HANDLE(radv_device, device, _device); radv_destroy_pipeline_binary(device, pAllocator, pipeline_binary); } VKAPI_ATTR VkResult VKAPI_CALL radv_GetPipelineBinaryDataKHR(VkDevice _device, const VkPipelineBinaryDataInfoKHR *pInfo, VkPipelineBinaryKeyKHR *pPipelineBinaryKey, size_t *pPipelineBinaryDataSize, void *pPipelineBinaryData) { VK_FROM_HANDLE(radv_pipeline_binary, pipeline_binary, pInfo->pipelineBinary); const size_t size = pipeline_binary->size; memcpy(pPipelineBinaryKey->key, pipeline_binary->key, sizeof(pipeline_binary->key)); pPipelineBinaryKey->keySize = sizeof(pipeline_binary->key); if (!pPipelineBinaryData) { *pPipelineBinaryDataSize = size; return VK_SUCCESS; } if (*pPipelineBinaryDataSize < size) { *pPipelineBinaryDataSize = size; return VK_ERROR_NOT_ENOUGH_SPACE_KHR; } memcpy(pPipelineBinaryData, pipeline_binary->data, size); *pPipelineBinaryDataSize = size; return VK_SUCCESS; } VKAPI_ATTR VkResult VKAPI_CALL radv_ReleaseCapturedPipelineDataKHR(VkDevice _device, const VkReleaseCapturedPipelineDataInfoKHR *pInfo, const VkAllocationCallbacks *pAllocator) { /* no-op */ return VK_SUCCESS; }