Edit
IABSD.fr/xenocara/lib/mesa/src/intel/vulkan/anv_sparse.c
jsg
- lib/mesa/src/intel/vulkan/anv_sparse.c
-
/* * Copyright © 2022 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include <anv_private.h> /* Sparse binding handling. * * There is one main structure passed around all over this file: * * - struct anv_sparse_binding_data: every resource (VkBuffer or VkImage) has * a pointer to an instance of this structure. It contains the virtual * memory address (VMA) used by the binding operations (which is different * from the VMA used by the anv_bo it's bound to) and the VMA range size. We * do not keep record of our our list of bindings (which ranges were bound * to which buffers). */ __attribute__((format(printf, 1, 2))) static void sparse_debug(const char *format, ...) { if (!INTEL_DEBUG(DEBUG_SPARSE)) return; va_list args; va_start(args, format); vfprintf(stderr, format, args); va_end(args); } static void dump_anv_vm_bind(struct anv_device *device, const struct anv_vm_bind *bind) { sparse_debug("[%s] ", bind->op == ANV_VM_BIND ? " bind " : "unbind"); if (bind->bo) sparse_debug("bo:%04u ", bind->bo->gem_handle); else sparse_debug("bo:---- "); sparse_debug("address:%016"PRIx64" size:%08"PRIx64" " "mem_offset:%08"PRIx64"\n", bind->address, bind->size, bind->bo_offset); } static void dump_anv_image(struct anv_image *i) { if (!INTEL_DEBUG(DEBUG_SPARSE)) return; sparse_debug("anv_image:\n"); sparse_debug("- format: %d\n", i->vk.format); sparse_debug("- extent: [%d, %d, %d]\n", i->vk.extent.width, i->vk.extent.height, i->vk.extent.depth); sparse_debug("- mip_levels: %d array_layers: %d samples: %d\n", i->vk.mip_levels, i->vk.array_layers, i->vk.samples); sparse_debug("- n_planes: %d\n", i->n_planes); sparse_debug("- disjoint: %d\n", i->disjoint); } static void dump_isl_surf(struct isl_surf *s) { if (!INTEL_DEBUG(DEBUG_SPARSE)) return; sparse_debug("isl_surf:\n"); const char *dim_s = s->dim == ISL_SURF_DIM_1D ? "1D" : s->dim == ISL_SURF_DIM_2D ? "2D" : s->dim == ISL_SURF_DIM_3D ? "3D" : "(ERROR)"; sparse_debug("- dim: %s\n", dim_s); sparse_debug("- tiling: %d (%s)\n", s->tiling, isl_tiling_to_name(s->tiling)); sparse_debug("- format: %s\n", isl_format_get_short_name(s->format)); sparse_debug("- image_alignment_el: [%d, %d, %d]\n", s->image_alignment_el.w, s->image_alignment_el.h, s->image_alignment_el.d); sparse_debug("- logical_level0_px: [%d, %d, %d, %d]\n", s->logical_level0_px.w, s->logical_level0_px.h, s->logical_level0_px.d, s->logical_level0_px.a); sparse_debug("- phys_level0_sa: [%d, %d, %d, %d]\n", s->phys_level0_sa.w, s->phys_level0_sa.h, s->phys_level0_sa.d, s->phys_level0_sa.a); sparse_debug("- levels: %d samples: %d\n", s->levels, s->samples); sparse_debug("- size_B: %"PRIu64" alignment_B: %u\n", s->size_B, s->alignment_B); sparse_debug("- row_pitch_B: %u\n", s->row_pitch_B); sparse_debug("- array_pitch_el_rows: %u\n", s->array_pitch_el_rows); const struct isl_format_layout *layout = isl_format_get_layout(s->format); sparse_debug("- format layout:\n"); sparse_debug(" - format:%d bpb:%d bw:%d bh:%d bd:%d\n", layout->format, layout->bpb, layout->bw, layout->bh, layout->bd); struct isl_tile_info tile_info; isl_surf_get_tile_info(s, &tile_info); sparse_debug("- tile info:\n"); sparse_debug(" - format_bpb: %d\n", tile_info.format_bpb); sparse_debug(" - logical_extent_el: [%d, %d, %d, %d]\n", tile_info.logical_extent_el.w, tile_info.logical_extent_el.h, tile_info.logical_extent_el.d, tile_info.logical_extent_el.a); sparse_debug(" - phys_extent_B: [%d, %d]\n", tile_info.phys_extent_B.w, tile_info.phys_extent_B.h); } static VkOffset3D vk_offset3d_px_to_el(const VkOffset3D offset_px, const struct isl_format_layout *layout) { return (VkOffset3D) { .x = offset_px.x / layout->bw, .y = offset_px.y / layout->bh, .z = offset_px.z / layout->bd, }; } static VkOffset3D vk_offset3d_el_to_px(const VkOffset3D offset_el, const struct isl_format_layout *layout) { return (VkOffset3D) { .x = offset_el.x * layout->bw, .y = offset_el.y * layout->bh, .z = offset_el.z * layout->bd, }; } static VkExtent3D vk_extent3d_px_to_el(const VkExtent3D extent_px, const struct isl_format_layout *layout) { return (VkExtent3D) { .width = extent_px.width / layout->bw, .height = extent_px.height / layout->bh, .depth = extent_px.depth / layout->bd, }; } static VkExtent3D vk_extent3d_el_to_px(const VkExtent3D extent_el, const struct isl_format_layout *layout) { return (VkExtent3D) { .width = extent_el.width * layout->bw, .height = extent_el.height * layout->bh, .depth = extent_el.depth * layout->bd, }; } static bool isl_tiling_supports_standard_block_shapes(enum isl_tiling tiling) { return isl_tiling_is_64(tiling) || tiling == ISL_TILING_ICL_Ys || tiling == ISL_TILING_SKL_Ys; } static uint32_t isl_calc_tile_size(struct isl_tile_info *tile_info) { uint32_t tile_size = tile_info->phys_extent_B.w * tile_info->phys_extent_B.h; assert(tile_size == 64 * 1024 || tile_size == 4096 || tile_size == 1); return tile_size; } static const VkExtent3D block_shapes_2d_1sample[] = { /* 8 bits: */ { .width = 256, .height = 256, .depth = 1 }, /* 16 bits: */ { .width = 256, .height = 128, .depth = 1 }, /* 32 bits: */ { .width = 128, .height = 128, .depth = 1 }, /* 64 bits: */ { .width = 128, .height = 64, .depth = 1 }, /* 128 bits: */ { .width = 64, .height = 64, .depth = 1 }, }; static const VkExtent3D block_shapes_3d_1sample[] = { /* 8 bits: */ { .width = 64, .height = 32, .depth = 32 }, /* 16 bits: */ { .width = 32, .height = 32, .depth = 32 }, /* 32 bits: */ { .width = 32, .height = 32, .depth = 16 }, /* 64 bits: */ { .width = 32, .height = 16, .depth = 16 }, /* 128 bits: */ { .width = 16, .height = 16, .depth = 16 }, }; static const VkExtent3D block_shapes_2d_2samples[] = { /* 8 bits: */ { .width = 128, .height = 256, .depth = 1 }, /* 16 bits: */ { .width = 128, .height = 128, .depth = 1 }, /* 32 bits: */ { .width = 64, .height = 128, .depth = 1 }, /* 64 bits: */ { .width = 64, .height = 64, .depth = 1 }, /* 128 bits: */ { .width = 32, .height = 64, .depth = 1 }, }; static const VkExtent3D block_shapes_2d_4samples[] = { /* 8 bits: */ { .width = 128, .height = 128, .depth = 1 }, /* 16 bits: */ { .width = 128, .height = 64, .depth = 1 }, /* 32 bits: */ { .width = 64, .height = 64, .depth = 1 }, /* 64 bits: */ { .width = 64, .height = 32, .depth = 1 }, /* 128 bits: */ { .width = 32, .height = 32, .depth = 1 }, }; static const VkExtent3D block_shapes_2d_8samples[] = { /* 8 bits: */ { .width = 64, .height = 128, .depth = 1 }, /* 16 bits: */ { .width = 64, .height = 64, .depth = 1 }, /* 32 bits: */ { .width = 32, .height = 64, .depth = 1 }, /* 64 bits: */ { .width = 32, .height = 32, .depth = 1 }, /* 128 bits: */ { .width = 16, .height = 32, .depth = 1 }, }; static const VkExtent3D block_shapes_2d_16samples[] = { /* 8 bits: */ { .width = 64, .height = 64, .depth = 1 }, /* 16 bits: */ { .width = 64, .height = 32, .depth = 1 }, /* 32 bits: */ { .width = 32, .height = 32, .depth = 1 }, /* 64 bits: */ { .width = 32, .height = 16, .depth = 1 }, /* 128 bits: */ { .width = 16, .height = 16, .depth = 1 }, }; static VkExtent3D anv_sparse_get_standard_image_block_shape(enum isl_format format, VkImageType image_type, VkSampleCountFlagBits samples, uint16_t texel_size) { const struct isl_format_layout *layout = isl_format_get_layout(format); VkExtent3D block_shape = { .width = 0, .height = 0, .depth = 0 }; int table_idx = ffs(texel_size) - 4; switch (samples) { case VK_SAMPLE_COUNT_1_BIT: switch (image_type) { case VK_IMAGE_TYPE_1D: /* 1D images don't have a standard block format. */ assert(false); break; case VK_IMAGE_TYPE_2D: block_shape = block_shapes_2d_1sample[table_idx]; break; case VK_IMAGE_TYPE_3D: block_shape = block_shapes_3d_1sample[table_idx]; break; default: fprintf(stderr, "unexpected image_type %d\n", image_type); assert(false); } break; case VK_SAMPLE_COUNT_2_BIT: block_shape = block_shapes_2d_2samples[table_idx]; break; case VK_SAMPLE_COUNT_4_BIT: block_shape = block_shapes_2d_4samples[table_idx]; break; case VK_SAMPLE_COUNT_8_BIT: block_shape = block_shapes_2d_8samples[table_idx]; break; case VK_SAMPLE_COUNT_16_BIT: block_shape = block_shapes_2d_16samples[table_idx]; break; default: fprintf(stderr, "unexpected sample count: %d\n", samples); assert(false); } return vk_extent3d_el_to_px(block_shape, layout); } /* Adds "bind_op" to the list in "submit", while also trying to check if we * can just extend the last operation instead. */ static VkResult anv_sparse_submission_add(struct anv_device *device, struct anv_sparse_submission *submit, struct anv_vm_bind *bind_op) { struct anv_vm_bind *prev_bind = submit->binds_len == 0 ? NULL : &submit->binds[submit->binds_len - 1]; if (prev_bind && bind_op->op == prev_bind->op && bind_op->bo == prev_bind->bo && bind_op->address == prev_bind->address + prev_bind->size && (bind_op->bo_offset == prev_bind->bo_offset + prev_bind->size || prev_bind->bo == NULL)) { prev_bind->size += bind_op->size; return VK_SUCCESS; } if (submit->binds_len < submit->binds_capacity) { submit->binds[submit->binds_len++] = *bind_op; return VK_SUCCESS; } int new_capacity = MAX2(32, submit->binds_capacity * 2); struct anv_vm_bind *new_binds = vk_realloc(&device->vk.alloc, submit->binds, new_capacity * sizeof(*new_binds), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (!new_binds) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); new_binds[submit->binds_len] = *bind_op; submit->binds = new_binds; submit->binds_len++; submit->binds_capacity = new_capacity; return VK_SUCCESS; } /* We really want to try to have all the page tables on as few BOs as possible * to benefit from cache locality and to keep the i915.ko relocation lists * small. On the other hand, we don't want to waste memory on unused space. */ #define ANV_TRTT_PAGE_TABLE_BO_SIZE (2 * 1024 * 1024) static VkResult trtt_make_page_table_bo(struct anv_device *device, struct anv_bo **bo) { VkResult result; struct anv_trtt *trtt = &device->trtt; result = anv_device_alloc_bo(device, "trtt-page-table", ANV_TRTT_PAGE_TABLE_BO_SIZE, ANV_BO_ALLOC_INTERNAL, 0 /* explicit_address */, bo); if (result != VK_SUCCESS) return result; if (trtt->num_page_table_bos < trtt->page_table_bos_capacity) { trtt->page_table_bos[trtt->num_page_table_bos++] = *bo; } else { int new_capacity = MAX2(8, trtt->page_table_bos_capacity * 2); struct anv_bo **new_page_table_bos = vk_realloc(&device->vk.alloc, trtt->page_table_bos, new_capacity * sizeof(*trtt->page_table_bos), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (!new_page_table_bos) { anv_device_release_bo(device, *bo); return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); } new_page_table_bos[trtt->num_page_table_bos] = *bo; trtt->page_table_bos = new_page_table_bos; trtt->page_table_bos_capacity = new_capacity; trtt->num_page_table_bos++; } trtt->cur_page_table_bo = *bo; trtt->next_page_table_bo_offset = 0; sparse_debug("new number of page table BOs: %d\n", trtt->num_page_table_bos); return VK_SUCCESS; } static VkResult trtt_get_page_table_bo(struct anv_device *device, struct anv_bo **bo, uint64_t *bo_addr) { struct anv_trtt *trtt = &device->trtt; VkResult result; if (!trtt->cur_page_table_bo) { result = trtt_make_page_table_bo(device, bo); if (result != VK_SUCCESS) return result; } *bo = trtt->cur_page_table_bo; *bo_addr = trtt->cur_page_table_bo->offset + trtt->next_page_table_bo_offset; trtt->next_page_table_bo_offset += 4096; if (trtt->next_page_table_bo_offset >= ANV_TRTT_PAGE_TABLE_BO_SIZE) trtt->cur_page_table_bo = NULL; return VK_SUCCESS; } /* For L3 and L2 pages, null and invalid entries are indicated by bits 1 and 0 * respectively. For L1 entries, the hardware compares the addresses against * what we program to the GFX_TRTT_NULL and GFX_TRTT_INVAL registers. */ #define ANV_TRTT_L3L2_NULL_ENTRY (1 << 1) #define ANV_TRTT_L3L2_INVALID_ENTRY (1 << 0) static void anv_trtt_bind_list_add_entry(struct util_dynarray *binds, uint64_t pte_addr, uint64_t entry_addr) { struct anv_trtt_bind b = { .pte_addr = pte_addr, .entry_addr = entry_addr, }; util_dynarray_append(binds, struct anv_trtt_bind, b); } /* Adds elements to the anv_trtt_bind structs passed. This doesn't write the * entries to the HW yet. */ static VkResult anv_trtt_bind_add(struct anv_device *device, uint64_t trtt_addr, uint64_t dest_addr, struct util_dynarray *l3l2_binds, struct util_dynarray *l1_binds) { VkResult result = VK_SUCCESS; struct anv_trtt *trtt = &device->trtt; bool is_null_bind = dest_addr == ANV_TRTT_L1_NULL_TILE_VAL; int l3_index = (trtt_addr >> 35) & 0x1FF; int l2_index = (trtt_addr >> 26) & 0x1FF; int l1_index = (trtt_addr >> 16) & 0x3FF; uint64_t l2_addr = trtt->l3_mirror[l3_index]; if (l2_addr == ANV_TRTT_L3L2_NULL_ENTRY && is_null_bind) { return VK_SUCCESS; } else if (l2_addr == 0 || l2_addr == ANV_TRTT_L3L2_NULL_ENTRY) { if (is_null_bind) { trtt->l3_mirror[l3_index] = ANV_TRTT_L3L2_NULL_ENTRY; anv_trtt_bind_list_add_entry(l3l2_binds, trtt->l3_addr + l3_index * sizeof(uint64_t), ANV_TRTT_L3L2_NULL_ENTRY); return VK_SUCCESS; } struct anv_bo *l2_bo; result = trtt_get_page_table_bo(device, &l2_bo, &l2_addr); if (result != VK_SUCCESS) return result; trtt->l3_mirror[l3_index] = l2_addr; anv_trtt_bind_list_add_entry(l3l2_binds, trtt->l3_addr + l3_index * sizeof(uint64_t), l2_addr); /* We have just created a new L2 table. Other resources may already have * been pointing to this L2 table relying on the fact that it was marked * as NULL, so now we need to mark every one of its entries as NULL in * order to preserve behavior for those entries. */ if (!util_dynarray_ensure_cap(l3l2_binds, l3l2_binds->capacity + 512 * sizeof(struct anv_trtt_bind))) return VK_ERROR_OUT_OF_HOST_MEMORY; for (int i = 0; i < 512; i++) { if (i != l2_index) { trtt->l2_mirror[l3_index * 512 + i] = ANV_TRTT_L3L2_NULL_ENTRY; anv_trtt_bind_list_add_entry(l3l2_binds, l2_addr + i * sizeof(uint64_t), ANV_TRTT_L3L2_NULL_ENTRY); } } } assert(l2_addr != 0 && l2_addr != ANV_TRTT_L3L2_NULL_ENTRY); /* The first page in the l2_mirror corresponds to l3_index=0 and so on. */ uint64_t l1_addr = trtt->l2_mirror[l3_index * 512 + l2_index]; if (l1_addr == ANV_TRTT_L3L2_NULL_ENTRY && is_null_bind) { return VK_SUCCESS; } else if (l1_addr == 0 || l1_addr == ANV_TRTT_L3L2_NULL_ENTRY) { if (is_null_bind) { trtt->l2_mirror[l3_index * 512 + l2_index] = ANV_TRTT_L3L2_NULL_ENTRY; anv_trtt_bind_list_add_entry(l3l2_binds, l2_addr + l2_index * sizeof(uint64_t), ANV_TRTT_L3L2_NULL_ENTRY); return VK_SUCCESS; } struct anv_bo *l1_bo; result = trtt_get_page_table_bo(device, &l1_bo, &l1_addr); if (result != VK_SUCCESS) return result; trtt->l2_mirror[l3_index * 512 + l2_index] = l1_addr; anv_trtt_bind_list_add_entry(l3l2_binds, l2_addr + l2_index * sizeof(uint64_t), l1_addr); } assert(l1_addr != 0 && l1_addr != ANV_TRTT_L3L2_NULL_ENTRY); anv_trtt_bind_list_add_entry(l1_binds, l1_addr + l1_index * sizeof(uint32_t), dest_addr); return VK_SUCCESS; } VkResult anv_sparse_trtt_garbage_collect_batches(struct anv_device *device, bool wait_completion) { struct anv_trtt *trtt = &device->trtt; uint64_t last_value; if (!wait_completion) { VkResult result = vk_sync_get_value(&device->vk, trtt->timeline, &last_value); if (result != VK_SUCCESS) return result; /* Valgrind doesn't know that drmSyncobjQuery writes to 'last_value' on * success. */ VG(VALGRIND_MAKE_MEM_DEFINED(&last_value, sizeof(last_value))); } else { last_value = trtt->timeline_val; } list_for_each_entry_safe(struct anv_trtt_submission, submit, &trtt->in_flight_batches, link) { if (submit->base.signal.signal_value <= last_value) { list_del(&submit->link); anv_async_submit_fini(&submit->base); vk_free(&device->vk.alloc, submit); continue; } if (!wait_completion) break; VkResult result = vk_sync_wait( &device->vk, submit->base.signal.sync, submit->base.signal.signal_value, VK_SYNC_WAIT_COMPLETE, os_time_get_absolute_timeout(OS_TIMEOUT_INFINITE)); if (result == VK_SUCCESS) { list_del(&submit->link); anv_async_submit_fini(&submit->base); vk_free(&device->vk.alloc, submit); continue; } /* If the wait failed but the caller wanted completion, return the * error. */ return result; } return VK_SUCCESS; } /* On success, this function initializes 'submit' and submits it, but doesn't * wait or free it. This allows the caller to submit multiple queues at the * same time before starting to wait for anything to complete. * If the function fails, the caller doesn't need to wait or fini anything, * just whatever other submissions may have succeeded in the past. */ static VkResult anv_trtt_first_bind_init_queue(struct anv_queue *queue, struct anv_async_submit *submit, bool init_l3_table, struct anv_bo *l3_bo) { struct anv_device *device = queue->device; struct anv_trtt *trtt = &device->trtt; VkResult result; result = anv_async_submit_init(submit, queue, &device->batch_bo_pool, false, true); if (result != VK_SUCCESS) return result; result = anv_genX(device->info, init_trtt_context_state)(submit); if (result != VK_SUCCESS) goto out_submit_fini; /* We only need to do this once, so pick the first queue. */ if (init_l3_table) { struct anv_trtt_bind l3l2_binds_data[512]; struct util_dynarray l3l2_binds; util_dynarray_init_from_stack(&l3l2_binds, l3l2_binds_data, sizeof(l3l2_binds_data)); for (int entry = 0; entry < 512; entry++) { trtt->l3_mirror[entry] = ANV_TRTT_L3L2_NULL_ENTRY; anv_trtt_bind_list_add_entry(&l3l2_binds, trtt->l3_addr + entry * sizeof(uint64_t), ANV_TRTT_L3L2_NULL_ENTRY); } anv_genX(device->info, write_trtt_entries)( submit, l3l2_binds.data, util_dynarray_num_elements(&l3l2_binds, struct anv_trtt_bind), NULL, 0); result = anv_reloc_list_add_bo(&submit->relocs, l3_bo); if (result != VK_SUCCESS) goto out_submit_fini; } anv_genX(device->info, async_submit_end)(submit); result = device->kmd_backend->queue_exec_async(submit, 0, NULL, 1, &submit->signal); if (result != VK_SUCCESS) goto out_submit_fini; /* If we succeed, it's our caller that's going to call * anv_async_submit_fini(). We do this so we can start waiting for the * submissions only after all the submissions are submitted. */ return VK_SUCCESS; out_submit_fini: /* If we fail, undo everything this function has done so the caller has * nothing to free. */ anv_async_submit_fini(submit); return result; } /* There are lots of applications that request for sparse binding to be * enabled but never use it, so we choose to delay the initialization of TR-TT * until the moment we know we're going to need it. */ static VkResult anv_trtt_first_bind_init(struct anv_device *device) { struct anv_trtt *trtt = &device->trtt; VkResult result = VK_SUCCESS; /* TR-TT submission needs a queue even when the API entry point doesn't * provide one, such as resource creation. We pick this queue from the user * created queues at init_device_state() under anv_CreateDevice. * * It is technically possible for the user to create sparse resources even * when they don't have a sparse queue: they won't be able to bind the * resource but they should still be able to use the resource and rely on * its unbound behavior. We haven't spotted any real world application or * even test suite that exercises this behavior. * * For now let's just print an error message and return, which means that * resource creation will succeed but the behavior will be undefined if the * resource is used, which goes against our claim that we support the * sparseResidencyNonResidentStrict property. * * TODO: be fully spec-compliant here. Maybe have a device-internal queue * independent of the application's queues for the TR-TT operations. */ if (unlikely(!trtt->queue)) { static bool warned = false; if (unlikely(!warned)) { fprintf(stderr, "FIXME: application has created a sparse resource " "but no queues capable of binding sparse resources were " "created. Using these resources will result in undefined " "behavior.\n"); warned = true; } return VK_SUCCESS; } simple_mtx_lock(&trtt->mutex); /* This means we have already initialized the first bind. */ if (likely(trtt->l3_addr)) { simple_mtx_unlock(&trtt->mutex); return VK_SUCCESS; } struct anv_async_submit submits[device->queue_count]; struct anv_bo *l3_bo; result = trtt_get_page_table_bo(device, &l3_bo, &trtt->l3_addr); if (result != VK_SUCCESS) goto out; trtt->l3_mirror = vk_zalloc(&device->vk.alloc, 4096, 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (!trtt->l3_mirror) { result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto out; } /* L3 has 512 entries, so we can have up to 512 L2 tables. */ trtt->l2_mirror = vk_zalloc(&device->vk.alloc, 512 * 4096, 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (!trtt->l2_mirror) { vk_free(&device->vk.alloc, trtt->l3_mirror); result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto out; } int n_submits; for (n_submits = 0; n_submits < device->queue_count; n_submits++) { result = anv_trtt_first_bind_init_queue(&device->queues[n_submits], &submits[n_submits], n_submits == 0, l3_bo); if (result != VK_SUCCESS) break; } for (uint32_t i = 0; i < n_submits; i++) { anv_async_submit_wait(&submits[i]); anv_async_submit_fini(&submits[i]); } out: if (result != VK_SUCCESS) trtt->l3_addr = 0; simple_mtx_unlock(&trtt->mutex); return result; } static VkResult anv_sparse_bind_trtt(struct anv_device *device, struct anv_sparse_submission *sparse_submit) { struct anv_trtt *trtt = &device->trtt; VkResult result; /* See the same check at anv_trtt_first_bind_init(). */ if (unlikely(!trtt->queue)) return VK_SUCCESS; if (!sparse_submit->queue) sparse_submit->queue = trtt->queue; struct anv_trtt_submission *submit = vk_zalloc(&device->vk.alloc, sizeof(*submit), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (submit == NULL) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); result = anv_async_submit_init(&submit->base, sparse_submit->queue, &device->batch_bo_pool, false, false); if (result != VK_SUCCESS) goto out_async; simple_mtx_lock(&trtt->mutex); /* Do this so we can avoid reallocs later. */ int l1_binds_capacity = 0; for (int b = 0; b < sparse_submit->binds_len; b++) { assert(sparse_submit->binds[b].size % (64 * 1024) == 0); int pages = sparse_submit->binds[b].size / (64 * 1024); l1_binds_capacity += pages; } /* Turn a series of virtual address maps, into a list of L3/L2/L1 TRTT page * table updates. */ /* These are arrays of struct anv_trtt_bind. */ struct util_dynarray l3l2_binds = {}; struct util_dynarray l1_binds; if (l1_binds_capacity <= 32) { size_t alloc_size = l1_binds_capacity * sizeof(struct anv_trtt_bind); struct anv_trtt_bind *ptr = alloca(alloc_size); util_dynarray_init_from_stack(&l1_binds, ptr, alloc_size); } else { util_dynarray_init(&l1_binds, NULL); if (!util_dynarray_ensure_cap(&l1_binds, l1_binds_capacity * sizeof(struct anv_trtt_bind))) goto out_dynarrays; } for (int b = 0; b < sparse_submit->binds_len; b++) { struct anv_vm_bind *vm_bind = &sparse_submit->binds[b]; for (uint64_t i = 0; i < vm_bind->size; i += 64 * 1024) { uint64_t trtt_addr = vm_bind->address + i; uint64_t dest_addr = (vm_bind->op == ANV_VM_BIND && vm_bind->bo) ? vm_bind->bo->offset + vm_bind->bo_offset + i : ANV_TRTT_L1_NULL_TILE_VAL; result = anv_trtt_bind_add(device, trtt_addr, dest_addr, &l3l2_binds, &l1_binds); if (result != VK_SUCCESS) goto out_dynarrays; } } /* Convert the L3/L2/L1 TRTT page table updates in anv_trtt_bind elements * into MI commands. */ uint32_t n_l3l2_binds = util_dynarray_num_elements(&l3l2_binds, struct anv_trtt_bind); uint32_t n_l1_binds = util_dynarray_num_elements(&l1_binds, struct anv_trtt_bind); sparse_debug("trtt_binds: num_vm_binds:%02d l3l2:%04d l1:%04d\n", sparse_submit->binds_len, n_l3l2_binds, n_l1_binds); /* This is not an error, the application is simply trying to reset state * that was already there. */ if (n_l3l2_binds == 0 && n_l1_binds == 0 && sparse_submit->wait_count == 0 && sparse_submit->signal_count == 0) goto out_dynarrays; anv_genX(device->info, write_trtt_entries)(&submit->base, l3l2_binds.data, n_l3l2_binds, l1_binds.data, n_l1_binds); util_dynarray_fini(&l1_binds); util_dynarray_fini(&l3l2_binds); anv_genX(device->info, async_submit_end)(&submit->base); if (submit->base.batch.status != VK_SUCCESS) { result = submit->base.batch.status; goto out_add_bind; } /* Add all the BOs backing TRTT page tables to the reloc list. */ if (device->physical->uses_relocs) { for (int i = 0; i < trtt->num_page_table_bos; i++) { result = anv_reloc_list_add_bo(&submit->base.relocs, trtt->page_table_bos[i]); if (result != VK_SUCCESS) goto out_add_bind; } } anv_sparse_trtt_garbage_collect_batches(device, false); submit->base.signal = (struct vk_sync_signal) { .sync = trtt->timeline, .signal_value = ++trtt->timeline_val, }; result = device->kmd_backend->queue_exec_async(&submit->base, sparse_submit->wait_count, sparse_submit->waits, sparse_submit->signal_count, sparse_submit->signals); if (result != VK_SUCCESS) { trtt->timeline_val--; goto out_add_bind; } list_addtail(&submit->link, &trtt->in_flight_batches); simple_mtx_unlock(&trtt->mutex); ANV_RMV(vm_binds, device, sparse_submit->binds, sparse_submit->binds_len); return VK_SUCCESS; out_dynarrays: util_dynarray_fini(&l1_binds); util_dynarray_fini(&l3l2_binds); out_add_bind: simple_mtx_unlock(&trtt->mutex); anv_async_submit_fini(&submit->base); out_async: vk_free(&device->vk.alloc, submit); return result; } static VkResult anv_sparse_bind_vm_bind(struct anv_device *device, struct anv_sparse_submission *submit) { struct anv_queue *queue = submit->queue; VkResult result = device->kmd_backend->vm_bind(device, submit, ANV_VM_BIND_FLAG_NONE); if (!queue) { assert(submit->wait_count == 0 && submit->signal_count == 0 && submit->binds_len == 1); return result; } if (result == VK_ERROR_OUT_OF_HOST_MEMORY) { /* If we get this, the system is under memory pressure. First we * manually wait for all our dependency syncobjs hoping that some memory * will be released while we wait, then we try to issue each bind * operation in a single ioctl as it requires less Kernel memory and so * we may be able to move things forward, although slowly, while also * waiting for each operation to complete before issuing the next. * Performance isn't a concern at this point: we're just trying to move * progress forward without crashing until whatever is eating too much * memory goes away. */ result = vk_sync_wait_many(&device->vk, submit->wait_count, submit->waits, VK_SYNC_WAIT_COMPLETE, INT64_MAX); if (result != VK_SUCCESS) return vk_queue_set_lost(&queue->vk, "vk_sync_wait_many failed"); struct vk_sync *sync; result = vk_sync_create(&device->vk, &device->physical->sync_syncobj_type, VK_SYNC_IS_TIMELINE, 0 /* initial_value */, &sync); if (result != VK_SUCCESS) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); for (int b = 0; b < submit->binds_len; b++) { struct vk_sync_signal sync_signal = { .sync = sync, .signal_value = b + 1, }; struct anv_sparse_submission s = { .queue = submit->queue, .binds = &submit->binds[b], .binds_len = 1, .binds_capacity = 1, .wait_count = 0, .signal_count = 1, .waits = NULL, .signals = &sync_signal, }; result = device->kmd_backend->vm_bind(device, &s, ANV_VM_BIND_FLAG_NONE); if (result != VK_SUCCESS) { vk_sync_destroy(&device->vk, sync); return vk_error(device, result); /* Well, at least we tried... */ } result = vk_sync_wait(&device->vk, sync, sync_signal.signal_value, VK_SYNC_WAIT_COMPLETE, UINT64_MAX); if (result != VK_SUCCESS) { vk_sync_destroy(&device->vk, sync); return vk_queue_set_lost(&queue->vk, "vk_sync_wait failed"); } } vk_sync_destroy(&device->vk, sync); for (uint32_t i = 0; i < submit->signal_count; i++) { struct vk_sync_signal *s = &submit->signals[i]; result = vk_sync_signal(&device->vk, s->sync, s->signal_value); if (result != VK_SUCCESS) return vk_queue_set_lost(&queue->vk, "vk_sync_signal failed"); } } return VK_SUCCESS; } VkResult anv_sparse_bind(struct anv_device *device, struct anv_sparse_submission *submit) { if (INTEL_DEBUG(DEBUG_SPARSE)) { for (int b = 0; b < submit->binds_len; b++) dump_anv_vm_bind(device, &submit->binds[b]); } return device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT ? anv_sparse_bind_trtt(device, submit) : anv_sparse_bind_vm_bind(device, submit); } VkResult anv_init_sparse_bindings(struct anv_device *device, uint64_t size_, struct anv_sparse_binding_data *sparse, enum anv_bo_alloc_flags alloc_flags, uint64_t client_address, struct anv_address *out_address) { VkResult result; uint64_t size = align64(size_, ANV_SPARSE_BLOCK_SIZE); if (device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT) alloc_flags |= ANV_BO_ALLOC_TRTT; sparse->address = anv_vma_alloc(device, size, ANV_SPARSE_BLOCK_SIZE, alloc_flags, intel_48b_address(client_address), &sparse->vma_heap); sparse->size = size; out_address->bo = NULL; out_address->offset = sparse->address; if (device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT) { result = anv_trtt_first_bind_init(device); if (result != VK_SUCCESS) goto out_vma_free; } else { struct anv_vm_bind bind = { .bo = NULL, /* That's a NULL binding. */ .address = sparse->address, .bo_offset = 0, .size = size, .op = ANV_VM_BIND, }; struct anv_sparse_submission submit = { .queue = NULL, .binds = &bind, .binds_len = 1, .binds_capacity = 1, .wait_count = 0, .signal_count = 0, }; result = anv_sparse_bind(device, &submit); if (result != VK_SUCCESS) goto out_vma_free; } p_atomic_inc(&device->num_sparse_resources); return VK_SUCCESS; out_vma_free: anv_vma_free(device, sparse->vma_heap, sparse->address, sparse->size); return result; } void anv_free_sparse_bindings(struct anv_device *device, struct anv_sparse_binding_data *sparse) { if (!sparse->address) return; sparse_debug("%s: address:0x%016"PRIx64" size:0x%08"PRIx64"\n", __func__, sparse->address, sparse->size); p_atomic_dec(&device->num_sparse_resources); struct anv_vm_bind unbind = { .bo = 0, .address = sparse->address, .bo_offset = 0, .size = sparse->size, .op = ANV_VM_UNBIND, }; struct anv_sparse_submission submit = { .queue = NULL, .binds = &unbind, .binds_len = 1, .binds_capacity = 1, .wait_count = 0, .signal_count = 0, }; VkResult res = anv_sparse_bind(device, &submit); /* Our callers don't have a way to signal failure to the upper layers, so * just keep the vma if we fail to unbind it. Still, let's have an * assertion because this really shouldn't be happening. */ assert(res == VK_SUCCESS); if (res != VK_SUCCESS) return; anv_vma_free(device, sparse->vma_heap, sparse->address, sparse->size); } static VkExtent3D anv_sparse_calc_block_shape(struct anv_physical_device *pdevice, struct isl_surf *surf, const struct isl_tile_info *tile_info) { const struct isl_format_layout *layout = isl_format_get_layout(surf->format); VkExtent3D block_shape_el = { .width = tile_info->logical_extent_el.width, .height = tile_info->logical_extent_el.height, .depth = tile_info->logical_extent_el.depth, }; VkExtent3D block_shape_px = vk_extent3d_el_to_px(block_shape_el, layout); assert(surf->tiling != ISL_TILING_LINEAR); return block_shape_px; } VkSparseImageFormatProperties anv_sparse_calc_image_format_properties(struct anv_physical_device *pdevice, VkImageAspectFlags aspect, VkImageType vk_image_type, VkSampleCountFlagBits vk_samples, struct isl_surf *surf) { const struct isl_format_layout *isl_layout = isl_format_get_layout(surf->format); struct isl_tile_info tile_info; isl_surf_get_tile_info(surf, &tile_info); const int bpb = isl_layout->bpb; assert(bpb == 8 || bpb == 16 || bpb == 32 || bpb == 64 ||bpb == 128); VkExtent3D granularity = anv_sparse_calc_block_shape(pdevice, surf, &tile_info); bool is_standard = false; bool is_known_nonstandard_format = false; /* We shouldn't be able to reach this function with a 1D image. */ assert(vk_image_type != VK_IMAGE_TYPE_1D); VkExtent3D std_shape = anv_sparse_get_standard_image_block_shape(surf->format, vk_image_type, vk_samples, bpb); /* YUV formats don't work with Tile64, which is required if we want to * claim standard block shapes. The spec requires us to support all * non-compressed color formats that non-sparse supports, so we can't just * say YUV formats are not supported by Sparse. So we end supporting this * format and anv_sparse_calc_miptail_properties() will say that everything * is part of the miptail. * * For more details on the hardware restriction, please check * isl_gfx125_filter_tiling(). */ if (pdevice->info.verx10 >= 125 && isl_format_is_yuv(surf->format)) is_known_nonstandard_format = true; /* The standard block shapes (and by extension, the tiling formats they * require) are simply incompatible with getting a 2D view of a 3D image. */ if (surf->usage & ISL_SURF_USAGE_2D_3D_COMPATIBLE_BIT) is_known_nonstandard_format = true; is_standard = granularity.width == std_shape.width && granularity.height == std_shape.height && granularity.depth == std_shape.depth; /* TODO: dEQP seems to care about the block shapes being standard even for * the cases where is_known_nonstandard_format is true. Luckily as of today * all of those cases are NotSupported but sooner or later we may end up * getting a failure. * Notice that in practice we report these cases as having the mip tail * starting on mip level 0, so the reported block shapes are irrelevant * since non-opaque binds are not supported. Still, dEQP seems to care. */ assert(is_standard || is_known_nonstandard_format); assert(!(is_standard && is_known_nonstandard_format)); bool wrong_block_size = isl_calc_tile_size(&tile_info) != ANV_SPARSE_BLOCK_SIZE; return (VkSparseImageFormatProperties) { .aspectMask = aspect, .imageGranularity = granularity, .flags = ((is_standard || is_known_nonstandard_format) ? 0 : VK_SPARSE_IMAGE_FORMAT_NONSTANDARD_BLOCK_SIZE_BIT) | (wrong_block_size ? VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT : 0), }; } /* The miptail is supposed to be this region where the tiniest mip levels * are squished together in one single page, which should save us some memory. * It's a hardware feature which our hardware supports on certain tiling * formats - the ones we always want to use for sparse resources. * * For sparse, the main feature of the miptail is that it only supports opaque * binds, so you either bind the whole miptail or you bind nothing at all, * there are no subresources inside it to separately bind. While the idea is * that the miptail as reported by sparse should match what our hardware does, * in practice we can say in our sparse functions that certain mip levels are * part of the miptail while from the point of view of our hardwared they * aren't. * * If we detect we're using the sparse-friendly tiling formats and ISL * supports miptails for them, we can just trust the miptail level set by ISL * and things can proceed as The Spec intended. * * However, if that's not the case, we have to go on a best-effort policy. We * could simply declare that every mip level is part of the miptail and be * done, but since that kinda defeats the purpose of Sparse we try to find * what level we really should be reporting as the first miptail level based * on the alignments of the surface subresources. */ void anv_sparse_calc_miptail_properties(struct anv_device *device, struct anv_image *image, VkImageAspectFlags vk_aspect, uint32_t *imageMipTailFirstLod, VkDeviceSize *imageMipTailSize, VkDeviceSize *imageMipTailOffset, VkDeviceSize *imageMipTailStride) { const uint32_t plane = anv_image_aspect_to_plane(image, vk_aspect); struct isl_surf *surf = &image->planes[plane].primary_surface.isl; uint64_t binding_plane_offset = image->planes[plane].primary_surface.memory_range.offset; struct isl_tile_info tile_info; isl_surf_get_tile_info(surf, &tile_info); uint64_t layer1_offset; uint32_t x_off, y_off; /* Treat the whole thing as a single miptail. We should have already * reported this image as VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT. * * In theory we could try to make ISL massage the alignments so that we * could at least claim mip level 0 to be not part of the miptail, but * that could end up wasting a lot of memory, so it's better to do * nothing and focus our efforts into making things use the appropriate * tiling formats that give us the standard block shapes. */ if (isl_calc_tile_size(&tile_info) != ANV_SPARSE_BLOCK_SIZE) goto out_everything_is_miptail; assert(surf->tiling != ISL_TILING_LINEAR); if (image->vk.array_layers == 1) { layer1_offset = surf->size_B; } else { isl_surf_get_image_offset_B_tile_sa(surf, 0, 1, 0, &layer1_offset, &x_off, &y_off); if (x_off || y_off) goto out_everything_is_miptail; } assert(layer1_offset % ANV_SPARSE_BLOCK_SIZE == 0); /* We could try to do better here, but there's not really any point since * we should be supporting the appropriate tiling formats everywhere. */ if (!isl_tiling_supports_standard_block_shapes(surf->tiling)) goto out_everything_is_miptail; int miptail_first_level = surf->miptail_start_level; if (miptail_first_level >= image->vk.mip_levels) goto out_no_miptail; uint64_t miptail_offset = 0; isl_surf_get_image_offset_B_tile_sa(surf, miptail_first_level, 0, 0, &miptail_offset, &x_off, &y_off); assert(x_off == 0 && y_off == 0); assert(miptail_offset % ANV_SPARSE_BLOCK_SIZE == 0); *imageMipTailFirstLod = miptail_first_level; *imageMipTailSize = ANV_SPARSE_BLOCK_SIZE; *imageMipTailOffset = binding_plane_offset + miptail_offset; *imageMipTailStride = layer1_offset; goto out_debug; out_no_miptail: *imageMipTailFirstLod = image->vk.mip_levels; *imageMipTailSize = 0; *imageMipTailOffset = 0; *imageMipTailStride = 0; goto out_debug; out_everything_is_miptail: *imageMipTailFirstLod = 0; *imageMipTailSize = surf->size_B; *imageMipTailOffset = binding_plane_offset; *imageMipTailStride = 0; out_debug: sparse_debug("miptail first_lod:%d size:%"PRIu64" offset:%"PRIu64" " "stride:%"PRIu64"\n", *imageMipTailFirstLod, *imageMipTailSize, *imageMipTailOffset, *imageMipTailStride); } static struct anv_vm_bind vk_bind_to_anv_vm_bind(struct anv_sparse_binding_data *sparse, const struct VkSparseMemoryBind *vk_bind) { struct anv_vm_bind anv_bind = { .bo = NULL, .address = sparse->address + vk_bind->resourceOffset, .bo_offset = 0, .size = vk_bind->size, .op = ANV_VM_BIND, }; assert(vk_bind->size); assert(vk_bind->resourceOffset + vk_bind->size <= sparse->size); if (vk_bind->memory != VK_NULL_HANDLE) { anv_bind.bo = anv_device_memory_from_handle(vk_bind->memory)->bo; anv_bind.bo_offset = vk_bind->memoryOffset, assert(vk_bind->memoryOffset + vk_bind->size <= anv_bind.bo->size); } return anv_bind; } static VkResult anv_sparse_bind_resource_memory(struct anv_device *device, struct anv_sparse_binding_data *sparse, uint64_t resource_size, const VkSparseMemoryBind *vk_bind, struct anv_sparse_submission *submit) { struct anv_vm_bind bind = vk_bind_to_anv_vm_bind(sparse, vk_bind); uint64_t rem = vk_bind->size % ANV_SPARSE_BLOCK_SIZE; if (rem != 0) { if (vk_bind->resourceOffset + vk_bind->size == resource_size) bind.size += ANV_SPARSE_BLOCK_SIZE - rem; else return vk_error(device, VK_ERROR_VALIDATION_FAILED_EXT); } return anv_sparse_submission_add(device, submit, &bind); } VkResult anv_sparse_bind_buffer(struct anv_device *device, struct anv_buffer *buffer, const VkSparseMemoryBind *vk_bind, struct anv_sparse_submission *submit) { return anv_sparse_bind_resource_memory(device, &buffer->sparse_data, buffer->vk.size, vk_bind, submit); } VkResult anv_sparse_bind_image_opaque(struct anv_device *device, struct anv_image *image, const VkSparseMemoryBind *vk_bind, struct anv_sparse_submission *submit) { struct anv_image_binding *b = &image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN]; assert(!image->disjoint); if (INTEL_DEBUG(DEBUG_SPARSE)) { sparse_debug("%s:\n", __func__); dump_anv_image(image); u_foreach_bit(b, image->vk.aspects) { VkImageAspectFlagBits aspect = 1 << b; const uint32_t plane = anv_image_aspect_to_plane(image, aspect); struct isl_surf *surf = &image->planes[plane].primary_surface.isl; sparse_debug("aspect 0x%x (plane %d):\n", aspect, plane); dump_isl_surf(surf); } sparse_debug("\n"); } return anv_sparse_bind_resource_memory(device, &b->sparse_data, b->memory_range.size, vk_bind, submit); } VkResult anv_sparse_bind_image_memory(struct anv_queue *queue, struct anv_image *image, const VkSparseImageMemoryBind *bind, struct anv_sparse_submission *submit) { struct anv_device *device = queue->device; VkImageAspectFlags aspect = bind->subresource.aspectMask; uint32_t mip_level = bind->subresource.mipLevel; uint32_t array_layer = bind->subresource.arrayLayer; assert(!(bind->flags & VK_SPARSE_MEMORY_BIND_METADATA_BIT)); struct anv_image_binding *img_binding = image->disjoint ? &image->bindings[anv_image_aspect_to_binding(image, aspect)] : &image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN]; struct anv_sparse_binding_data *sparse_data = &img_binding->sparse_data; const uint32_t plane = anv_image_aspect_to_plane(image, aspect); struct isl_surf *surf = &image->planes[plane].primary_surface.isl; uint64_t binding_plane_offset = image->planes[plane].primary_surface.memory_range.offset; const struct isl_format_layout *layout = isl_format_get_layout(surf->format); struct isl_tile_info tile_info; isl_surf_get_tile_info(surf, &tile_info); if (INTEL_DEBUG(DEBUG_SPARSE)) { sparse_debug("%s:\n", __func__); sparse_debug("mip_level:%d array_layer:%d\n", mip_level, array_layer); sparse_debug("aspect:0x%x plane:%d\n", aspect, plane); sparse_debug("binding offset: [%d, %d, %d] extent: [%d, %d, %d]\n", bind->offset.x, bind->offset.y, bind->offset.z, bind->extent.width, bind->extent.height, bind->extent.depth); dump_anv_image(image); dump_isl_surf(surf); sparse_debug("\n"); } VkExtent3D block_shape_px = anv_sparse_calc_block_shape(device->physical, surf, &tile_info); VkExtent3D block_shape_el = vk_extent3d_px_to_el(block_shape_px, layout); /* Both bind->offset and bind->extent are in pixel units. */ VkOffset3D bind_offset_el = vk_offset3d_px_to_el(bind->offset, layout); /* The spec says we only really need to align if for a given coordinate * offset + extent equals the corresponding dimensions of the image * subresource, but all the other non-aligned usage is invalid, so just * align everything. */ VkExtent3D bind_extent_px = { .width = ALIGN_NPOT(bind->extent.width, block_shape_px.width), .height = ALIGN_NPOT(bind->extent.height, block_shape_px.height), .depth = ALIGN_NPOT(bind->extent.depth, block_shape_px.depth), }; VkExtent3D bind_extent_el = vk_extent3d_px_to_el(bind_extent_px, layout); /* Nothing that has a tile_size different than ANV_SPARSE_BLOCK_SIZE should * be reaching here, as these cases should be treated as "everything is * part of the miptail" (see anv_sparse_calc_miptail_properties()). */ assert(isl_calc_tile_size(&tile_info) == ANV_SPARSE_BLOCK_SIZE); /* How many blocks are necessary to form a whole line on this image? */ const uint32_t blocks_per_line = surf->row_pitch_B / (layout->bpb / 8) / block_shape_el.width; /* The loop below will try to bind a whole line of blocks at a time as * they're guaranteed to be contiguous, so we calculate how many blocks * that is and how big is each block to figure the bind size of a whole * line. */ uint64_t line_bind_size_in_blocks = bind_extent_el.width / block_shape_el.width; uint64_t line_bind_size = line_bind_size_in_blocks * ANV_SPARSE_BLOCK_SIZE; assert(line_bind_size_in_blocks != 0); assert(line_bind_size != 0); uint64_t memory_offset = bind->memoryOffset; for (uint32_t z = bind_offset_el.z; z < bind_offset_el.z + bind_extent_el.depth; z += block_shape_el.depth) { uint64_t subresource_offset_B; uint32_t subresource_x_offset, subresource_y_offset; isl_surf_get_image_offset_B_tile_sa(surf, mip_level, array_layer, z, &subresource_offset_B, &subresource_x_offset, &subresource_y_offset); assert(subresource_x_offset == 0 && subresource_y_offset == 0); assert(subresource_offset_B % ANV_SPARSE_BLOCK_SIZE == 0); for (uint32_t y = bind_offset_el.y; y < bind_offset_el.y + bind_extent_el.height; y+= block_shape_el.height) { uint32_t line_block_offset = y / block_shape_el.height * blocks_per_line; uint64_t line_start_B = subresource_offset_B + line_block_offset * ANV_SPARSE_BLOCK_SIZE; uint64_t bind_offset_B = line_start_B + (bind_offset_el.x / block_shape_el.width) * ANV_SPARSE_BLOCK_SIZE; VkSparseMemoryBind opaque_bind = { .resourceOffset = binding_plane_offset + bind_offset_B, .size = line_bind_size, .memory = bind->memory, .memoryOffset = memory_offset, .flags = bind->flags, }; memory_offset += line_bind_size; assert(line_start_B % ANV_SPARSE_BLOCK_SIZE == 0); assert(opaque_bind.resourceOffset % ANV_SPARSE_BLOCK_SIZE == 0); assert(opaque_bind.size % ANV_SPARSE_BLOCK_SIZE == 0); struct anv_vm_bind anv_bind = vk_bind_to_anv_vm_bind(sparse_data, &opaque_bind); VkResult result = anv_sparse_submission_add(device, submit, &anv_bind); if (result != VK_SUCCESS) return result; } } return VK_SUCCESS; } VkResult anv_sparse_image_check_support(struct anv_physical_device *pdevice, VkImageCreateFlags flags, VkImageTiling tiling, VkSampleCountFlagBits samples, VkImageType type, VkFormat vk_format) { assert(flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT); /* The spec says: * "A sparse image created using VK_IMAGE_CREATE_SPARSE_BINDING_BIT (but * not VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) supports all formats that * non-sparse usage supports, and supports both VK_IMAGE_TILING_OPTIMAL * and VK_IMAGE_TILING_LINEAR tiling." */ if (!(flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)) return VK_SUCCESS; if (type == VK_IMAGE_TYPE_1D) return VK_ERROR_FORMAT_NOT_SUPPORTED; /* From here on, these are the rules: * "A sparse image created using VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT * supports all non-compressed color formats with power-of-two element * size that non-sparse usage supports. Additional formats may also be * supported and can be queried via * vkGetPhysicalDeviceSparseImageFormatProperties. * VK_IMAGE_TILING_LINEAR tiling is not supported." */ /* We choose not to support sparse residency on emulated compressed * formats due to the additional image plane. It would make the * implementation extremely complicated. */ if (anv_is_format_emulated(pdevice, vk_format)) return VK_ERROR_FORMAT_NOT_SUPPORTED; /* While the spec itself says linear is not supported (see above), deqp-vk * tries anyway to create linear sparse images, so we have to check for it. * This is also said in VUID-VkImageCreateInfo-tiling-04121: * "If tiling is VK_IMAGE_TILING_LINEAR, flags must not contain * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT" */ if (tiling == VK_IMAGE_TILING_LINEAR) return VK_ERROR_FORMAT_NOT_SUPPORTED; if ((samples & VK_SAMPLE_COUNT_2_BIT && !pdevice->vk.supported_features.sparseResidency2Samples) || (samples & VK_SAMPLE_COUNT_4_BIT && !pdevice->vk.supported_features.sparseResidency4Samples) || (samples & VK_SAMPLE_COUNT_8_BIT && !pdevice->vk.supported_features.sparseResidency8Samples) || (samples & VK_SAMPLE_COUNT_16_BIT && !pdevice->vk.supported_features.sparseResidency16Samples) || samples & VK_SAMPLE_COUNT_32_BIT || samples & VK_SAMPLE_COUNT_64_BIT) return VK_ERROR_FEATURE_NOT_PRESENT; /* While the Vulkan spec allows us to support depth/stencil sparse images * everywhere, sometimes we're not able to have them with the tiling * formats that give us the standard block shapes. Having standard block * shapes is higher priority than supporting depth/stencil sparse images. * * Please see ISL's filter_tiling() functions for accurate explanations on * why depth/stencil images are not always supported with the tiling * formats we want. But in short: depth/stencil support in our HW is * limited to 2D and we can't build a 2D view of a 3D image with these * tiling formats due to the address swizzling being different. */ VkImageAspectFlags aspects = vk_format_aspects(vk_format); if (aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) { /* For multi-sampled images, the image layouts for color and * depth/stencil are different, and only the color layout is compatible * with the standard block shapes. */ if (samples != VK_SAMPLE_COUNT_1_BIT) return VK_ERROR_FORMAT_NOT_SUPPORTED; /* For 125+, isl_gfx125_filter_tiling() claims 3D is not supported. * For the previous platforms, isl_gfx6_filter_tiling() says only 2D is * supported. */ if (pdevice->info.verx10 >= 125) { if (type == VK_IMAGE_TYPE_3D) return VK_ERROR_FORMAT_NOT_SUPPORTED; } else { if (type != VK_IMAGE_TYPE_2D) return VK_ERROR_FORMAT_NOT_SUPPORTED; } } const struct anv_format *anv_format = anv_get_format(pdevice, vk_format); if (!anv_format) return VK_ERROR_FORMAT_NOT_SUPPORTED; for (int p = 0; p < anv_format->n_planes; p++) { enum isl_format isl_format = anv_format->planes[p].isl_format; if (isl_format == ISL_FORMAT_UNSUPPORTED) return VK_ERROR_FORMAT_NOT_SUPPORTED; const struct isl_format_layout *isl_layout = isl_format_get_layout(isl_format); /* As quoted above, we only need to support the power-of-two formats. * The problem with the non-power-of-two formats is that we need an * integer number of pixels to fit into a sparse block, so we'd need the * sparse block sizes to be, for example, 192k for 24bpp. * * TODO: add support for these formats. */ if (isl_layout->bpb != 8 && isl_layout->bpb != 16 && isl_layout->bpb != 32 && isl_layout->bpb != 64 && isl_layout->bpb != 128) return VK_ERROR_FORMAT_NOT_SUPPORTED; /* ISL_TILING_64_XE2_BIT's block shapes are not always Vulkan's standard * block shapes, so exclude what's non-standard. */ if (pdevice->info.ver == 20) { switch (samples) { case VK_SAMPLE_COUNT_2_BIT: if (isl_layout->bpb == 128) return VK_ERROR_FORMAT_NOT_SUPPORTED; break; case VK_SAMPLE_COUNT_8_BIT: if (isl_layout->bpb == 8 || isl_layout->bpb == 32) return VK_ERROR_FORMAT_NOT_SUPPORTED; break; case VK_SAMPLE_COUNT_16_BIT: if (isl_layout->bpb == 64) return VK_ERROR_FORMAT_NOT_SUPPORTED; break; default: break; } } } /* These YUV formats are considered by Vulkan to be compressed 2x1 blocks. * We don't need to support them since they're compressed. On Gfx12 we * can't even have Tile64 for them. Once we do support these formats we'll * have to report the correct block shapes because dEQP cares about them, * and we'll have to adjust for the fact that ISL treats these as 16bpp 1x1 * blocks instead of 32bpp 2x1 compressed blocks (as block shapes are * reported in units of compressed blocks). */ if (vk_format == VK_FORMAT_G8B8G8R8_422_UNORM || vk_format == VK_FORMAT_B8G8R8G8_422_UNORM) return VK_ERROR_FORMAT_NOT_SUPPORTED; return VK_SUCCESS; }