Edit
IABSD.fr/xenocara/lib/mesa/src/intel/compiler/brw_nir_rt_builder.h
jsg
- lib/mesa/src/intel/compiler/brw_nir_rt_builder.h
-
/* * Copyright © 2020 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #pragma once /* This file provides helpers to access memory based data structures that the * RT hardware reads/writes and their locations. * * See also "Memory Based Data Structures for Ray Tracing" (BSpec 47547) and * "Ray Tracing Address Computation for Memory Resident Structures" (BSpec * 47550). */ #include "brw_rt.h" #include "nir_builder.h" #define is_access_for_builder(b) \ ((b)->shader->info.stage == MESA_SHADER_FRAGMENT ? \ ACCESS_INCLUDE_HELPERS : 0) static inline nir_def * brw_nir_rt_load(nir_builder *b, nir_def *addr, unsigned align, unsigned components, unsigned bit_size) { return nir_build_load_global(b, components, bit_size, addr, .align_mul = align, .access = is_access_for_builder(b)); } static inline void brw_nir_rt_store(nir_builder *b, nir_def *addr, unsigned align, nir_def *value, unsigned write_mask) { nir_build_store_global(b, value, addr, .align_mul = align, .write_mask = (write_mask) & BITFIELD_MASK(value->num_components), .access = is_access_for_builder(b)); } static inline nir_def * brw_nir_rt_load_const(nir_builder *b, unsigned components, nir_def *addr) { return nir_load_global_constant_uniform_block_intel( b, components, 32, addr, .access = ACCESS_CAN_REORDER | ACCESS_NON_WRITEABLE, .align_mul = 64); } static inline nir_def * brw_load_btd_dss_id(nir_builder *b) { return nir_load_topology_id_intel(b, .base = BRW_TOPOLOGY_ID_DSS); } static inline nir_def * brw_load_eu_thread_simd(nir_builder *b) { return nir_load_topology_id_intel(b, .base = BRW_TOPOLOGY_ID_EU_THREAD_SIMD); } static inline nir_def * brw_nir_rt_async_stack_id(nir_builder *b) { return nir_iadd(b, nir_umul_32x16(b, nir_load_ray_num_dss_rt_stacks_intel(b), brw_load_btd_dss_id(b)), nir_load_btd_stack_id_intel(b)); } static inline nir_def * brw_nir_rt_sync_stack_id(nir_builder *b) { return brw_load_eu_thread_simd(b); } /* We have our own load/store scratch helpers because they emit a global * memory read or write based on the scratch_base_ptr system value rather * than a load/store_scratch intrinsic. */ static inline nir_def * brw_nir_rt_load_scratch(nir_builder *b, uint32_t offset, unsigned align, unsigned num_components, unsigned bit_size) { nir_def *addr = nir_iadd_imm(b, nir_load_scratch_base_ptr(b, 1, 64, 1), offset); return brw_nir_rt_load(b, addr, MIN2(align, BRW_BTD_STACK_ALIGN), num_components, bit_size); } static inline void brw_nir_rt_store_scratch(nir_builder *b, uint32_t offset, unsigned align, nir_def *value, nir_component_mask_t write_mask) { nir_def *addr = nir_iadd_imm(b, nir_load_scratch_base_ptr(b, 1, 64, 1), offset); brw_nir_rt_store(b, addr, MIN2(align, BRW_BTD_STACK_ALIGN), value, write_mask); } static inline void brw_nir_btd_spawn(nir_builder *b, nir_def *record_addr) { nir_btd_spawn_intel(b, nir_load_btd_global_arg_addr_intel(b), record_addr); } static inline void brw_nir_btd_retire(nir_builder *b) { nir_btd_retire_intel(b); } /** This is a pseudo-op which does a bindless return * * It loads the return address from the stack and calls btd_spawn to spawn the * resume shader. */ static inline void brw_nir_btd_return(struct nir_builder *b) { nir_def *resume_addr = brw_nir_rt_load_scratch(b, BRW_BTD_STACK_RESUME_BSR_ADDR_OFFSET, 8 /* align */, 1, 64); brw_nir_btd_spawn(b, resume_addr); } static inline void assert_def_size(nir_def *def, unsigned num_components, unsigned bit_size) { assert(def->num_components == num_components); assert(def->bit_size == bit_size); } static inline nir_def * brw_nir_num_rt_stacks(nir_builder *b, const struct intel_device_info *devinfo) { return nir_imul_imm(b, nir_load_ray_num_dss_rt_stacks_intel(b), intel_device_info_dual_subslice_id_bound(devinfo)); } static inline nir_def * brw_nir_rt_sw_hotzone_addr(nir_builder *b, const struct intel_device_info *devinfo) { nir_def *offset32 = nir_imul_imm(b, brw_nir_rt_async_stack_id(b), BRW_RT_SIZEOF_HOTZONE); offset32 = nir_iadd(b, offset32, nir_ineg(b, nir_imul_imm(b, brw_nir_num_rt_stacks(b, devinfo), BRW_RT_SIZEOF_HOTZONE))); return nir_iadd(b, nir_load_ray_base_mem_addr_intel(b), nir_i2i64(b, offset32)); } static inline nir_def * brw_nir_rt_sync_stack_addr(nir_builder *b, nir_def *base_mem_addr, nir_def *num_dss_rt_stacks) { /* Bspec 47547 (Xe) and 56936 (Xe2+) say: * For Ray queries (Synchronous Ray Tracing), the formula is similar but * goes down from rtMemBasePtr : * * syncBase = RTDispatchGlobals.rtMemBasePtr * - (DSSID * NUM_SIMD_LANES_PER_DSS + SyncStackID + 1) * * syncStackSize * * We assume that we can calculate a 32-bit offset first and then add it * to the 64-bit base address at the end. * * However, on HSD 14020275151 it's clarified that the HW uses * NUM_SYNC_STACKID_PER_DSS instead. */ nir_def *offset32 = nir_imul(b, nir_iadd(b, nir_imul(b, brw_load_btd_dss_id(b), num_dss_rt_stacks), nir_iadd_imm(b, brw_nir_rt_sync_stack_id(b), 1)), nir_imm_int(b, BRW_RT_SIZEOF_RAY_QUERY)); return nir_isub(b, base_mem_addr, nir_u2u64(b, offset32)); } static inline nir_def * brw_nir_rt_stack_addr(nir_builder *b) { /* From the BSpec "Address Computation for Memory Based Data Structures: * Ray and TraversalStack (Async Ray Tracing)": * * stackBase = RTDispatchGlobals.rtMemBasePtr * + (DSSID * RTDispatchGlobals.numDSSRTStacks + stackID) * * RTDispatchGlobals.stackSizePerRay // 64B aligned * * We assume that we can calculate a 32-bit offset first and then add it * to the 64-bit base address at the end. */ nir_def *offset32 = nir_imul(b, brw_nir_rt_async_stack_id(b), nir_load_ray_hw_stack_size_intel(b)); return nir_iadd(b, nir_load_ray_base_mem_addr_intel(b), nir_u2u64(b, offset32)); } static inline nir_def * brw_nir_rt_mem_hit_addr_from_addr(nir_builder *b, nir_def *stack_addr, bool committed) { return nir_iadd_imm(b, stack_addr, committed ? 0 : BRW_RT_SIZEOF_HIT_INFO); } static inline nir_def * brw_nir_rt_mem_hit_addr(nir_builder *b, bool committed) { return nir_iadd_imm(b, brw_nir_rt_stack_addr(b), committed ? 0 : BRW_RT_SIZEOF_HIT_INFO); } static inline nir_def * brw_nir_rt_hit_attrib_data_addr(nir_builder *b) { return nir_iadd_imm(b, brw_nir_rt_stack_addr(b), BRW_RT_OFFSETOF_HIT_ATTRIB_DATA); } static inline nir_def * brw_nir_rt_mem_ray_addr(nir_builder *b, nir_def *stack_addr, enum brw_rt_bvh_level bvh_level) { /* From the BSpec "Address Computation for Memory Based Data Structures: * Ray and TraversalStack (Async Ray Tracing)": * * rayBase = stackBase + sizeof(HitInfo) * 2 // 64B aligned * rayPtr = rayBase + bvhLevel * sizeof(Ray); // 64B aligned * * In Vulkan, we always have exactly two levels of BVH: World and Object. */ uint32_t offset = BRW_RT_SIZEOF_HIT_INFO * 2 + bvh_level * BRW_RT_SIZEOF_RAY; return nir_iadd_imm(b, stack_addr, offset); } static inline nir_def * brw_nir_rt_sw_stack_addr(nir_builder *b, const struct intel_device_info *devinfo) { nir_def *addr = nir_load_ray_base_mem_addr_intel(b); nir_def *offset32 = nir_imul(b, brw_nir_num_rt_stacks(b, devinfo), nir_load_ray_hw_stack_size_intel(b)); addr = nir_iadd(b, addr, nir_u2u64(b, offset32)); nir_def *offset_in_stack = nir_imul(b, nir_u2u64(b, brw_nir_rt_async_stack_id(b)), nir_u2u64(b, nir_load_ray_sw_stack_size_intel(b))); return nir_iadd(b, addr, offset_in_stack); } static inline nir_def * nir_unpack_64_4x16_split_z(nir_builder *b, nir_def *val) { return nir_unpack_32_2x16_split_x(b, nir_unpack_64_2x32_split_y(b, val)); } struct brw_nir_rt_globals_defs { nir_def *base_mem_addr; nir_def *call_stack_handler_addr; nir_def *hw_stack_size; nir_def *num_dss_rt_stacks; nir_def *hit_sbt_addr; nir_def *hit_sbt_stride; nir_def *miss_sbt_addr; nir_def *miss_sbt_stride; nir_def *sw_stack_size; nir_def *launch_size; nir_def *call_sbt_addr; nir_def *call_sbt_stride; nir_def *resume_sbt_addr; }; static inline void brw_nir_rt_load_globals_addr(nir_builder *b, struct brw_nir_rt_globals_defs *defs, nir_def *addr) { nir_def *data; data = brw_nir_rt_load_const(b, 16, addr); defs->base_mem_addr = nir_pack_64_2x32(b, nir_trim_vector(b, data, 2)); defs->call_stack_handler_addr = nir_pack_64_2x32(b, nir_channels(b, data, 0x3 << 2)); defs->hw_stack_size = nir_channel(b, data, 4); defs->num_dss_rt_stacks = nir_iand_imm(b, nir_channel(b, data, 5), 0xffff); defs->hit_sbt_addr = nir_pack_64_2x32_split(b, nir_channel(b, data, 8), nir_extract_i16(b, nir_channel(b, data, 9), nir_imm_int(b, 0))); defs->hit_sbt_stride = nir_unpack_32_2x16_split_y(b, nir_channel(b, data, 9)); defs->miss_sbt_addr = nir_pack_64_2x32_split(b, nir_channel(b, data, 10), nir_extract_i16(b, nir_channel(b, data, 11), nir_imm_int(b, 0))); defs->miss_sbt_stride = nir_unpack_32_2x16_split_y(b, nir_channel(b, data, 11)); defs->sw_stack_size = nir_channel(b, data, 12); defs->launch_size = nir_channels(b, data, 0x7u << 13); data = brw_nir_rt_load_const(b, 8, nir_iadd_imm(b, addr, 64)); defs->call_sbt_addr = nir_pack_64_2x32_split(b, nir_channel(b, data, 0), nir_extract_i16(b, nir_channel(b, data, 1), nir_imm_int(b, 0))); defs->call_sbt_stride = nir_unpack_32_2x16_split_y(b, nir_channel(b, data, 1)); defs->resume_sbt_addr = nir_pack_64_2x32(b, nir_channels(b, data, 0x3 << 2)); } static inline void brw_nir_rt_load_globals(nir_builder *b, struct brw_nir_rt_globals_defs *defs) { brw_nir_rt_load_globals_addr(b, defs, nir_load_btd_global_arg_addr_intel(b)); } static inline nir_def * brw_nir_rt_unpack_leaf_ptr(nir_builder *b, nir_def *vec2) { /* Hit record leaf pointers are 42-bit and assumed to be in 64B chunks. * This leaves 22 bits at the top for other stuff. */ nir_def *ptr64 = nir_imul_imm(b, nir_pack_64_2x32(b, vec2), 64); /* The top 16 bits (remember, we shifted by 6 already) contain garbage * that we need to get rid of. */ nir_def *ptr_lo = nir_unpack_64_2x32_split_x(b, ptr64); nir_def *ptr_hi = nir_unpack_64_2x32_split_y(b, ptr64); ptr_hi = nir_extract_i16(b, ptr_hi, nir_imm_int(b, 0)); return nir_pack_64_2x32_split(b, ptr_lo, ptr_hi); } /** * MemHit memory layout (BSpec 47547) : * * name bits description * - t 32 hit distance of current hit (or initial traversal distance) * - u 32 barycentric hit coordinates * - v 32 barycentric hit coordinates * - primIndexDelta 16 prim index delta for compressed meshlets and quads * - valid 1 set if there is a hit * - leafType 3 type of node primLeafPtr is pointing to * - primLeafIndex 4 index of the hit primitive inside the leaf * - bvhLevel 3 the instancing level at which the hit occured * - frontFace 1 whether we hit the front-facing side of a triangle (also used to pass opaque flag when calling intersection shaders) * - pad0 4 unused bits * - primLeafPtr 42 pointer to BVH leaf node (multiple of 64 bytes) * - hitGroupRecPtr0 22 LSB of hit group record of the hit triangle (multiple of 16 bytes) * - instLeafPtr 42 pointer to BVH instance leaf node (in multiple of 64 bytes) * - hitGroupRecPtr1 22 MSB of hit group record of the hit triangle (multiple of 32 bytes) */ struct brw_nir_rt_mem_hit_defs { nir_def *t; nir_def *tri_bary; /**< Only valid for triangle geometry */ nir_def *aabb_hit_kind; /**< Only valid for AABB geometry */ nir_def *valid; nir_def *leaf_type; nir_def *prim_index_delta; nir_def *prim_leaf_index; nir_def *bvh_level; nir_def *front_face; nir_def *done; /**< Only for ray queries */ nir_def *prim_leaf_ptr; nir_def *inst_leaf_ptr; }; static inline void brw_nir_rt_load_mem_hit_from_addr(nir_builder *b, struct brw_nir_rt_mem_hit_defs *defs, nir_def *stack_addr, bool committed) { nir_def *hit_addr = brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, committed); nir_def *data = brw_nir_rt_load(b, hit_addr, 16, 4, 32); defs->t = nir_channel(b, data, 0); defs->aabb_hit_kind = nir_channel(b, data, 1); defs->tri_bary = nir_channels(b, data, 0x6); nir_def *bitfield = nir_channel(b, data, 3); defs->prim_index_delta = nir_ubitfield_extract(b, bitfield, nir_imm_int(b, 0), nir_imm_int(b, 16)); defs->valid = nir_i2b(b, nir_iand_imm(b, bitfield, 1u << 16)); defs->leaf_type = nir_ubitfield_extract(b, bitfield, nir_imm_int(b, 17), nir_imm_int(b, 3)); defs->prim_leaf_index = nir_ubitfield_extract(b, bitfield, nir_imm_int(b, 20), nir_imm_int(b, 4)); defs->bvh_level = nir_ubitfield_extract(b, bitfield, nir_imm_int(b, 24), nir_imm_int(b, 3)); defs->front_face = nir_i2b(b, nir_iand_imm(b, bitfield, 1 << 27)); defs->done = nir_i2b(b, nir_iand_imm(b, bitfield, 1 << 28)); data = brw_nir_rt_load(b, nir_iadd_imm(b, hit_addr, 16), 16, 4, 32); defs->prim_leaf_ptr = brw_nir_rt_unpack_leaf_ptr(b, nir_channels(b, data, 0x3 << 0)); defs->inst_leaf_ptr = brw_nir_rt_unpack_leaf_ptr(b, nir_channels(b, data, 0x3 << 2)); } static inline void brw_nir_rt_load_mem_hit(nir_builder *b, struct brw_nir_rt_mem_hit_defs *defs, bool committed) { brw_nir_rt_load_mem_hit_from_addr(b, defs, brw_nir_rt_stack_addr(b), committed); } static inline void brw_nir_memcpy_global(nir_builder *b, nir_def *dst_addr, uint32_t dst_align, nir_def *src_addr, uint32_t src_align, uint32_t size) { /* We're going to copy in 16B chunks */ assert(size % 16 == 0); dst_align = MIN2(dst_align, 16); src_align = MIN2(src_align, 16); for (unsigned offset = 0; offset < size; offset += 16) { nir_def *data = brw_nir_rt_load(b, nir_iadd_imm(b, src_addr, offset), 16, 4, 32); brw_nir_rt_store(b, nir_iadd_imm(b, dst_addr, offset), 16, data, 0xf /* write_mask */); } } static inline void brw_nir_memclear_global(nir_builder *b, nir_def *dst_addr, uint32_t dst_align, uint32_t size) { /* We're going to copy in 16B chunks */ assert(size % 16 == 0); dst_align = MIN2(dst_align, 16); nir_def *zero = nir_imm_ivec4(b, 0, 0, 0, 0); for (unsigned offset = 0; offset < size; offset += 16) { brw_nir_rt_store(b, nir_iadd_imm(b, dst_addr, offset), dst_align, zero, 0xf /* write_mask */); } } static inline nir_def * brw_nir_rt_query_done(nir_builder *b, nir_def *stack_addr) { struct brw_nir_rt_mem_hit_defs hit_in = {}; brw_nir_rt_load_mem_hit_from_addr(b, &hit_in, stack_addr, false /* committed */); return hit_in.done; } static inline void brw_nir_rt_set_dword_bit_at(nir_builder *b, nir_def *addr, uint32_t addr_offset, uint32_t bit) { nir_def *dword_addr = nir_iadd_imm(b, addr, addr_offset); nir_def *dword = brw_nir_rt_load(b, dword_addr, 4, 1, 32); brw_nir_rt_store(b, dword_addr, 4, nir_ior_imm(b, dword, 1u << bit), 0x1); } static inline void brw_nir_rt_query_mark_done(nir_builder *b, nir_def *stack_addr) { brw_nir_rt_set_dword_bit_at(b, brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, false /* committed */), 4 * 3 /* dword offset */, 28 /* bit */); } /* This helper clears the 3rd dword of the MemHit structure where the valid * bit is located. */ static inline void brw_nir_rt_query_mark_init(nir_builder *b, nir_def *stack_addr) { nir_def *dword_addr; for (uint32_t i = 0; i < 2; i++) { dword_addr = nir_iadd_imm(b, brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, i == 0 /* committed */), 4 * 3 /* dword offset */); brw_nir_rt_store(b, dword_addr, 4, nir_imm_int(b, 0), 0x1); } } /* This helper is pretty much a memcpy of uncommitted into committed hit * structure, just adding the valid bit. */ static inline void brw_nir_rt_commit_hit_addr(nir_builder *b, nir_def *stack_addr) { nir_def *dst_addr = brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, true /* committed */); nir_def *src_addr = brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, false /* committed */); for (unsigned offset = 0; offset < BRW_RT_SIZEOF_HIT_INFO; offset += 16) { nir_def *data = brw_nir_rt_load(b, nir_iadd_imm(b, src_addr, offset), 16, 4, 32); if (offset == 0) { data = nir_vec4(b, nir_channel(b, data, 0), nir_channel(b, data, 1), nir_channel(b, data, 2), nir_ior_imm(b, nir_channel(b, data, 3), 0x1 << 16 /* valid */)); /* Also write the potential hit as we change it. */ brw_nir_rt_store(b, nir_iadd_imm(b, src_addr, offset), 16, data, 0xf /* write_mask */); } brw_nir_rt_store(b, nir_iadd_imm(b, dst_addr, offset), 16, data, 0xf /* write_mask */); } } static inline void brw_nir_rt_commit_hit(nir_builder *b) { nir_def *stack_addr = brw_nir_rt_stack_addr(b); brw_nir_rt_commit_hit_addr(b, stack_addr); } static inline void brw_nir_rt_generate_hit_addr(nir_builder *b, nir_def *stack_addr, nir_def *t_val) { nir_def *committed_addr = brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, true /* committed */); nir_def *potential_addr = brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, false /* committed */); /* Set: * * potential.t = t_val; * potential.valid = true; */ nir_def *potential_hit_dwords_0_3 = brw_nir_rt_load(b, potential_addr, 16, 4, 32); potential_hit_dwords_0_3 = nir_vec4(b, t_val, nir_channel(b, potential_hit_dwords_0_3, 1), nir_channel(b, potential_hit_dwords_0_3, 2), nir_ior_imm(b, nir_channel(b, potential_hit_dwords_0_3, 3), (0x1 << 16) /* valid */)); brw_nir_rt_store(b, potential_addr, 16, potential_hit_dwords_0_3, 0xf /* write_mask */); /* Set: * * committed.t = t_val; * committed.u = 0.0f; * committed.v = 0.0f; * committed.valid = true; * committed.leaf_type = potential.leaf_type; * committed.bvh_level = BRW_RT_BVH_LEVEL_OBJECT; * committed.front_face = false; * committed.prim_leaf_index = 0; * committed.done = false; */ nir_def *committed_hit_dwords_0_3 = brw_nir_rt_load(b, committed_addr, 16, 4, 32); committed_hit_dwords_0_3 = nir_vec4(b, t_val, nir_imm_float(b, 0.0f), nir_imm_float(b, 0.0f), nir_ior_imm(b, nir_ior_imm(b, nir_channel(b, potential_hit_dwords_0_3, 3), 0x000e0000), (0x1 << 16) /* valid */ | (BRW_RT_BVH_LEVEL_OBJECT << 24) /* leaf_type */)); brw_nir_rt_store(b, committed_addr, 16, committed_hit_dwords_0_3, 0xf /* write_mask */); /* Set: * * committed.prim_leaf_ptr = potential.prim_leaf_ptr; * committed.inst_leaf_ptr = potential.inst_leaf_ptr; */ brw_nir_memcpy_global(b, nir_iadd_imm(b, committed_addr, 16), 16, nir_iadd_imm(b, potential_addr, 16), 16, 16); } struct brw_nir_rt_mem_ray_defs { nir_def *orig; nir_def *dir; nir_def *t_near; nir_def *t_far; nir_def *root_node_ptr; nir_def *ray_flags; nir_def *hit_group_sr_base_ptr; nir_def *hit_group_sr_stride; nir_def *miss_sr_ptr; nir_def *shader_index_multiplier; nir_def *inst_leaf_ptr; nir_def *ray_mask; }; static inline void brw_nir_rt_store_mem_ray_query_at_addr(nir_builder *b, nir_def *ray_addr, const struct brw_nir_rt_mem_ray_defs *defs) { assert_def_size(defs->orig, 3, 32); assert_def_size(defs->dir, 3, 32); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 0), 16, nir_vec4(b, nir_channel(b, defs->orig, 0), nir_channel(b, defs->orig, 1), nir_channel(b, defs->orig, 2), nir_channel(b, defs->dir, 0)), ~0 /* write mask */); assert_def_size(defs->t_near, 1, 32); assert_def_size(defs->t_far, 1, 32); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 16), 16, nir_vec4(b, nir_channel(b, defs->dir, 1), nir_channel(b, defs->dir, 2), defs->t_near, defs->t_far), ~0 /* write mask */); assert_def_size(defs->root_node_ptr, 1, 64); assert_def_size(defs->ray_flags, 1, 16); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 32), 16, nir_vec2(b, nir_unpack_64_2x32_split_x(b, defs->root_node_ptr), nir_pack_32_2x16_split(b, nir_unpack_64_4x16_split_z(b, defs->root_node_ptr), defs->ray_flags)), 0x3 /* write mask */); /* leaf_ptr is optional */ nir_def *inst_leaf_ptr; if (defs->inst_leaf_ptr) { inst_leaf_ptr = defs->inst_leaf_ptr; } else { inst_leaf_ptr = nir_imm_int64(b, 0); } assert_def_size(inst_leaf_ptr, 1, 64); assert_def_size(defs->ray_mask, 1, 32); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 56), 8, nir_vec2(b, nir_unpack_64_2x32_split_x(b, inst_leaf_ptr), nir_pack_32_2x16_split(b, nir_unpack_64_4x16_split_z(b, inst_leaf_ptr), nir_unpack_32_2x16_split_x(b, defs->ray_mask))), ~0 /* write mask */); } static inline void brw_nir_rt_store_mem_ray(nir_builder *b, const struct brw_nir_rt_mem_ray_defs *defs, enum brw_rt_bvh_level bvh_level) { nir_def *ray_addr = brw_nir_rt_mem_ray_addr(b, brw_nir_rt_stack_addr(b), bvh_level); assert_def_size(defs->orig, 3, 32); assert_def_size(defs->dir, 3, 32); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 0), 16, nir_vec4(b, nir_channel(b, defs->orig, 0), nir_channel(b, defs->orig, 1), nir_channel(b, defs->orig, 2), nir_channel(b, defs->dir, 0)), ~0 /* write mask */); assert_def_size(defs->t_near, 1, 32); assert_def_size(defs->t_far, 1, 32); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 16), 16, nir_vec4(b, nir_channel(b, defs->dir, 1), nir_channel(b, defs->dir, 2), defs->t_near, defs->t_far), ~0 /* write mask */); assert_def_size(defs->root_node_ptr, 1, 64); assert_def_size(defs->ray_flags, 1, 16); assert_def_size(defs->hit_group_sr_base_ptr, 1, 64); assert_def_size(defs->hit_group_sr_stride, 1, 16); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 32), 16, nir_vec4(b, nir_unpack_64_2x32_split_x(b, defs->root_node_ptr), nir_pack_32_2x16_split(b, nir_unpack_64_4x16_split_z(b, defs->root_node_ptr), defs->ray_flags), nir_unpack_64_2x32_split_x(b, defs->hit_group_sr_base_ptr), nir_pack_32_2x16_split(b, nir_unpack_64_4x16_split_z(b, defs->hit_group_sr_base_ptr), defs->hit_group_sr_stride)), ~0 /* write mask */); /* leaf_ptr is optional */ nir_def *inst_leaf_ptr; if (defs->inst_leaf_ptr) { inst_leaf_ptr = defs->inst_leaf_ptr; } else { inst_leaf_ptr = nir_imm_int64(b, 0); } assert_def_size(defs->miss_sr_ptr, 1, 64); assert_def_size(defs->shader_index_multiplier, 1, 32); assert_def_size(inst_leaf_ptr, 1, 64); assert_def_size(defs->ray_mask, 1, 32); brw_nir_rt_store(b, nir_iadd_imm(b, ray_addr, 48), 16, nir_vec4(b, nir_unpack_64_2x32_split_x(b, defs->miss_sr_ptr), nir_pack_32_2x16_split(b, nir_unpack_64_4x16_split_z(b, defs->miss_sr_ptr), nir_unpack_32_2x16_split_x(b, nir_ishl(b, defs->shader_index_multiplier, nir_imm_int(b, 8)))), nir_unpack_64_2x32_split_x(b, inst_leaf_ptr), nir_pack_32_2x16_split(b, nir_unpack_64_4x16_split_z(b, inst_leaf_ptr), nir_unpack_32_2x16_split_x(b, defs->ray_mask))), ~0 /* write mask */); } static inline void brw_nir_rt_load_mem_ray_from_addr(nir_builder *b, struct brw_nir_rt_mem_ray_defs *defs, nir_def *ray_base_addr, enum brw_rt_bvh_level bvh_level) { nir_def *ray_addr = brw_nir_rt_mem_ray_addr(b, ray_base_addr, bvh_level); nir_def *data[4] = { brw_nir_rt_load(b, nir_iadd_imm(b, ray_addr, 0), 16, 4, 32), brw_nir_rt_load(b, nir_iadd_imm(b, ray_addr, 16), 16, 4, 32), brw_nir_rt_load(b, nir_iadd_imm(b, ray_addr, 32), 16, 4, 32), brw_nir_rt_load(b, nir_iadd_imm(b, ray_addr, 48), 16, 4, 32), }; defs->orig = nir_trim_vector(b, data[0], 3); defs->dir = nir_vec3(b, nir_channel(b, data[0], 3), nir_channel(b, data[1], 0), nir_channel(b, data[1], 1)); defs->t_near = nir_channel(b, data[1], 2); defs->t_far = nir_channel(b, data[1], 3); defs->root_node_ptr = nir_pack_64_2x32_split(b, nir_channel(b, data[2], 0), nir_extract_i16(b, nir_channel(b, data[2], 1), nir_imm_int(b, 0))); defs->ray_flags = nir_unpack_32_2x16_split_y(b, nir_channel(b, data[2], 1)); defs->hit_group_sr_base_ptr = nir_pack_64_2x32_split(b, nir_channel(b, data[2], 2), nir_extract_i16(b, nir_channel(b, data[2], 3), nir_imm_int(b, 0))); defs->hit_group_sr_stride = nir_unpack_32_2x16_split_y(b, nir_channel(b, data[2], 3)); defs->miss_sr_ptr = nir_pack_64_2x32_split(b, nir_channel(b, data[3], 0), nir_extract_i16(b, nir_channel(b, data[3], 1), nir_imm_int(b, 0))); defs->shader_index_multiplier = nir_ushr(b, nir_unpack_32_2x16_split_y(b, nir_channel(b, data[3], 1)), nir_imm_int(b, 8)); defs->inst_leaf_ptr = nir_pack_64_2x32_split(b, nir_channel(b, data[3], 2), nir_extract_i16(b, nir_channel(b, data[3], 3), nir_imm_int(b, 0))); defs->ray_mask = nir_unpack_32_2x16_split_y(b, nir_channel(b, data[3], 3)); } static inline void brw_nir_rt_load_mem_ray(nir_builder *b, struct brw_nir_rt_mem_ray_defs *defs, enum brw_rt_bvh_level bvh_level) { brw_nir_rt_load_mem_ray_from_addr(b, defs, brw_nir_rt_stack_addr(b), bvh_level); } struct brw_nir_rt_bvh_instance_leaf_defs { nir_def *shader_index; nir_def *contribution_to_hit_group_index; nir_def *world_to_object[4]; nir_def *instance_id; nir_def *instance_index; nir_def *object_to_world[4]; }; static inline void brw_nir_rt_load_bvh_instance_leaf(nir_builder *b, struct brw_nir_rt_bvh_instance_leaf_defs *defs, nir_def *leaf_addr) { nir_def *leaf_desc = brw_nir_rt_load(b, leaf_addr, 4, 2, 32); defs->shader_index = nir_iand_imm(b, nir_channel(b, leaf_desc, 0), (1 << 24) - 1); defs->contribution_to_hit_group_index = nir_iand_imm(b, nir_channel(b, leaf_desc, 1), (1 << 24) - 1); defs->world_to_object[0] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 16), 4, 3, 32); defs->world_to_object[1] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 28), 4, 3, 32); defs->world_to_object[2] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 40), 4, 3, 32); /* The last column of the matrices is swapped between the two probably * because it makes it easier/faster for hardware somehow. */ defs->object_to_world[3] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 52), 4, 3, 32); nir_def *data = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 64), 4, 4, 32); defs->instance_id = nir_channel(b, data, 2); defs->instance_index = nir_channel(b, data, 3); defs->object_to_world[0] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 80), 4, 3, 32); defs->object_to_world[1] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 92), 4, 3, 32); defs->object_to_world[2] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 104), 4, 3, 32); defs->world_to_object[3] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 116), 4, 3, 32); } struct brw_nir_rt_bvh_primitive_leaf_defs { nir_def *shader_index; nir_def *geom_mask; nir_def *geom_index; nir_def *type; nir_def *geom_flags; }; static inline void brw_nir_rt_load_bvh_primitive_leaf(nir_builder *b, struct brw_nir_rt_bvh_primitive_leaf_defs *defs, nir_def *leaf_addr) { nir_def *desc = brw_nir_rt_load(b, leaf_addr, 4, 2, 32); defs->shader_index = nir_ubitfield_extract(b, nir_channel(b, desc, 0), nir_imm_int(b, 23), nir_imm_int(b, 0)); defs->geom_mask = nir_ubitfield_extract(b, nir_channel(b, desc, 0), nir_imm_int(b, 31), nir_imm_int(b, 24)); defs->geom_index = nir_ubitfield_extract(b, nir_channel(b, desc, 1), nir_imm_int(b, 28), nir_imm_int(b, 0)); defs->type = nir_ubitfield_extract(b, nir_channel(b, desc, 1), nir_imm_int(b, 29), nir_imm_int(b, 29)); defs->geom_flags = nir_ubitfield_extract(b, nir_channel(b, desc, 1), nir_imm_int(b, 31), nir_imm_int(b, 30)); } struct brw_nir_rt_bvh_primitive_leaf_positions_defs { nir_def *positions[3]; }; static inline void brw_nir_rt_load_bvh_primitive_leaf_positions(nir_builder *b, struct brw_nir_rt_bvh_primitive_leaf_positions_defs *defs, nir_def *leaf_addr) { for (unsigned i = 0; i < ARRAY_SIZE(defs->positions); i++) { defs->positions[i] = brw_nir_rt_load(b, nir_iadd_imm(b, leaf_addr, 16 + i * 4 * 3), 4, 3, 32); } } static inline nir_def * brw_nir_rt_load_primitive_id_from_hit(nir_builder *b, nir_def *is_procedural, const struct brw_nir_rt_mem_hit_defs *defs) { if (!is_procedural) { is_procedural = nir_ieq_imm(b, defs->leaf_type, BRW_RT_BVH_NODE_TYPE_PROCEDURAL); } nir_def *prim_id_proc, *prim_id_quad; nir_push_if(b, is_procedural); { /* For procedural leafs, the index is in dw[3]. */ nir_def *offset = nir_iadd_imm(b, nir_ishl_imm(b, defs->prim_leaf_index, 2), 12); prim_id_proc = nir_load_global(b, nir_iadd(b, defs->prim_leaf_ptr, nir_u2u64(b, offset)), 4, /* align */ 1, 32); } nir_push_else(b, NULL); { /* For quad leafs, the index is dw[2] and there is a 16bit additional * offset in dw[3]. */ prim_id_quad = nir_load_global(b, nir_iadd_imm(b, defs->prim_leaf_ptr, 8), 4, /* align */ 1, 32); prim_id_quad = nir_iadd(b, prim_id_quad, defs->prim_index_delta); } nir_pop_if(b, NULL); return nir_if_phi(b, prim_id_proc, prim_id_quad); } static inline nir_def * brw_nir_rt_acceleration_structure_to_root_node(nir_builder *b, nir_def *as_addr) { /* The HW memory structure in which we specify what acceleration structure * to traverse, takes the address to the root node in the acceleration * structure, not the acceleration structure itself. To find that, we have * to read the root node offset from the acceleration structure which is * the first QWord. * * But if the acceleration structure pointer is NULL, then we should return * NULL as root node pointer. * * TODO: we could optimize this by assuming that for a given version of the * BVH, we can find the root node at a given offset. */ nir_def *root_node_ptr, *null_node_ptr; nir_push_if(b, nir_ieq_imm(b, as_addr, 0)); { null_node_ptr = nir_imm_int64(b, 0); } nir_push_else(b, NULL); { root_node_ptr = nir_iadd(b, as_addr, brw_nir_rt_load(b, as_addr, 256, 1, 64)); } nir_pop_if(b, NULL); return nir_if_phi(b, null_node_ptr, root_node_ptr); }