Edit
IABSD.fr/xenocara/lib/mesa/src/intel/compiler/brw_eu_compact.c
jsg
- lib/mesa/src/intel/compiler/brw_eu_compact.c
-
/* * Copyright © 2012-2018 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ /** @file * * Instruction compaction is a feature of G45 and newer hardware that allows * for a smaller instruction encoding. * * The instruction cache is on the order of 32KB, and many programs generate * far more instructions than that. The instruction cache is built to barely * keep up with instruction dispatch ability in cache hit cases -- L1 * instruction cache misses that still hit in the next level could limit * throughput by around 50%. * * The idea of instruction compaction is that most instructions use a tiny * subset of the GPU functionality, so we can encode what would be a 16 byte * instruction in 8 bytes using some lookup tables for various fields. * * * Instruction compaction capabilities vary subtly by generation. * * G45's support for instruction compaction is very limited. Jump counts on * this generation are in units of 16-byte uncompacted instructions. As such, * all jump targets must be 16-byte aligned. Also, all instructions must be * naturally aligned, i.e. uncompacted instructions must be 16-byte aligned. * A G45-only instruction, NENOP, must be used to provide padding to align * uncompacted instructions. * * Gfx5 removes these restrictions and changes jump counts to be in units of * 8-byte compacted instructions, allowing jump targets to be only 8-byte * aligned. Uncompacted instructions can also be placed on 8-byte boundaries. * * Gfx6 adds the ability to compact instructions with a limited range of * immediate values. Compactable immediates have 12 unrestricted bits, and a * 13th bit that's replicated through the high 20 bits, to create the 32-bit * value of DW3 in the uncompacted instruction word. * * On Gfx7 we can compact some control flow instructions with a small positive * immediate in the low bits of DW3, like ENDIF with the JIP field. Other * control flow instructions with UIP cannot be compacted, because of the * replicated 13th bit. No control flow instructions can be compacted on Gfx6 * since the jump count field is not in DW3. * * break JIP/UIP * cont JIP/UIP * halt JIP/UIP * if JIP/UIP * else JIP (plus UIP on BDW+) * endif JIP * while JIP (must be negative) * * Gen 8 adds support for compacting 3-src instructions. * * Gfx12 reduces the number of bits that available to compacted immediates from * 13 to 12, but improves the compaction of floating-point immediates by * allowing the high bits to be encoded (the sign, 8-bit exponent, and the * three most significant bits of the mantissa), rather than the lowest bits of * the mantissa. */ #include "brw_eu.h" #include "brw_disasm.h" #include "brw_disasm_info.h" #include "dev/intel_debug.h" static const uint16_t g45_subreg_table[32] = { 0b000000000000000, 0b000000010000000, 0b000001000000000, 0b000100000000000, 0b000000000100000, 0b100000000000000, 0b000000000010000, 0b001100000000000, 0b001010000000000, 0b000000100000000, 0b001000000000000, 0b000000000001000, 0b000000001000000, 0b000000000000001, 0b000010000000000, 0b000000010100000, 0b000000000000111, 0b000001000100000, 0b011000000000000, 0b000000110000000, 0b000000000000010, 0b000000000000100, 0b000000001100000, 0b000100000000010, 0b001110011000110, 0b001110100001000, 0b000110011000110, 0b000001000011000, 0b000110010000100, 0b001100000000110, 0b000000010000110, 0b000001000110000, }; static const uint32_t gfx8_control_index_table[32] = { 0b0000000000000000010, 0b0000100000000000000, 0b0000100000000000001, 0b0000100000000000010, 0b0000100000000000011, 0b0000100000000000100, 0b0000100000000000101, 0b0000100000000000111, 0b0000100000000001000, 0b0000100000000001001, 0b0000100000000001101, 0b0000110000000000000, 0b0000110000000000001, 0b0000110000000000010, 0b0000110000000000011, 0b0000110000000000100, 0b0000110000000000101, 0b0000110000000000111, 0b0000110000000001001, 0b0000110000000001101, 0b0000110000000010000, 0b0000110000100000000, 0b0001000000000000000, 0b0001000000000000010, 0b0001000000000000100, 0b0001000000100000000, 0b0010110000000000000, 0b0010110000000010000, 0b0011000000000000000, 0b0011000000100000000, 0b0101000000000000000, 0b0101000000100000000, }; static const uint32_t gfx8_datatype_table[32] = { 0b001000000000000000001, 0b001000000000001000000, 0b001000000000001000001, 0b001000000000011000001, 0b001000000000101011101, 0b001000000010111011101, 0b001000000011101000001, 0b001000000011101000101, 0b001000000011101011101, 0b001000001000001000001, 0b001000011000001000000, 0b001000011000001000001, 0b001000101000101000101, 0b001000111000101000100, 0b001000111000101000101, 0b001011100011101011101, 0b001011101011100011101, 0b001011101011101011100, 0b001011101011101011101, 0b001011111011101011100, 0b000000000010000001100, 0b001000000000001011101, 0b001000000000101000101, 0b001000001000001000000, 0b001000101000101000100, 0b001000111000100000100, 0b001001001001000001001, 0b001010111011101011101, 0b001011111011101011101, 0b001001111001101001100, 0b001001001001001001000, 0b001001011001001001000, }; static const uint16_t gfx8_subreg_table[32] = { 0b000000000000000, 0b000000000000001, 0b000000000001000, 0b000000000001111, 0b000000000010000, 0b000000010000000, 0b000000100000000, 0b000000110000000, 0b000001000000000, 0b000001000010000, 0b000001010000000, 0b001000000000000, 0b001000000000001, 0b001000010000001, 0b001000010000010, 0b001000010000011, 0b001000010000100, 0b001000010000111, 0b001000010001000, 0b001000010001110, 0b001000010001111, 0b001000110000000, 0b001000111101000, 0b010000000000000, 0b010000110000000, 0b011000000000000, 0b011110010000111, 0b100000000000000, 0b101000000000000, 0b110000000000000, 0b111000000000000, 0b111000000011100, }; static const uint16_t gfx8_src_index_table[32] = { 0b000000000000, 0b000000000010, 0b000000010000, 0b000000010010, 0b000000011000, 0b000000100000, 0b000000101000, 0b000001001000, 0b000001010000, 0b000001110000, 0b000001111000, 0b001100000000, 0b001100000010, 0b001100001000, 0b001100010000, 0b001100010010, 0b001100100000, 0b001100101000, 0b001100111000, 0b001101000000, 0b001101000010, 0b001101001000, 0b001101010000, 0b001101100000, 0b001101101000, 0b001101110000, 0b001101110001, 0b001101111000, 0b010001101000, 0b010001101001, 0b010001101010, 0b010110001000, }; static const uint32_t gfx11_datatype_table[32] = { 0b001000000000000000001, 0b001000000000001000000, 0b001000000000001000001, 0b001000000000011000001, 0b001000000000101100101, 0b001000000101111100101, 0b001000000100101000001, 0b001000000100101000101, 0b001000000100101100101, 0b001000001000001000001, 0b001000011000001000000, 0b001000011000001000001, 0b001000101000101000101, 0b001000111000101000100, 0b001000111000101000101, 0b001100100100101100101, 0b001100101100100100101, 0b001100101100101100100, 0b001100101100101100101, 0b001100111100101100100, 0b000000000010000001100, 0b001000000000001100101, 0b001000000000101000101, 0b001000001000001000000, 0b001000101000101000100, 0b001000111000100000100, 0b001001001001000001001, 0b001101111100101100101, 0b001100111100101100101, 0b001001111001101001100, 0b001001001001001001000, 0b001001011001001001000, }; static const uint32_t gfx12_control_index_table[32] = { 0b000000000000000000100, /* (16|M0) */ 0b000000000000000000011, /* (8|M0) */ 0b000000010000000000000, /* (W) (1|M0) */ 0b000000010000000000100, /* (W) (16|M0) */ 0b000000010000000000011, /* (W) (8|M0) */ 0b010000000000000000100, /* (16|M0) (ge)f0.0 */ 0b000000000000000100100, /* (16|M16) */ 0b010100000000000000100, /* (16|M0) (lt)f0.0 */ 0b000000000000000000000, /* (1|M0) */ 0b000010000000000000100, /* (16|M0) (sat) */ 0b000000000000000010011, /* (8|M8) */ 0b001100000000000000100, /* (16|M0) (gt)f0.0 */ 0b000100000000000000100, /* (16|M0) (eq)f0.0 */ 0b000100010000000000100, /* (W) (16|M0) (eq)f0.0 */ 0b001000000000000000100, /* (16|M0) (ne)f0.0 */ 0b000000000000100000100, /* (f0.0) (16|M0) */ 0b010100000000000000011, /* (8|M0) (lt)f0.0 */ 0b000000000000110000100, /* (f1.0) (16|M0) */ 0b000000010000000000001, /* (W) (2|M0) */ 0b000000000000101000100, /* (f0.1) (16|M0) */ 0b000000000000111000100, /* (f1.1) (16|M0) */ 0b010000010000000000100, /* (W) (16|M0) (ge)f0.0 */ 0b000000000000000100011, /* (8|M16) */ 0b000000000000000110011, /* (8|M24) */ 0b010100010000000000100, /* (W) (16|M0) (lt)f0.0 */ 0b010000000000000000011, /* (8|M0) (ge)f0.0 */ 0b000100010000000000000, /* (W) (1|M0) (eq)f0.0 */ 0b000010000000000000011, /* (8|M0) (sat) */ 0b010100000000010000100, /* (16|M0) (lt)f1.0 */ 0b000100000000000000011, /* (8|M0) (eq)f0.0 */ 0b000001000000000000011, /* (8|M0) {AccWrEn} */ 0b000000010000000100100, /* (W) (16|M16) */ }; static const uint32_t gfx12_datatype_table[32] = { 0b11010110100101010100, /* grf<1>:f grf:f grf:f */ 0b00000110100101010100, /* grf<1>:f grf:f arf:ub */ 0b00000010101101010100, /* grf<1>:f imm:f arf:ub */ 0b01010110110101010100, /* grf<1>:f grf:f imm:f */ 0b11010100100101010100, /* arf<1>:f grf:f grf:f */ 0b11010010100101010100, /* grf<1>:f arf:f grf:f */ 0b01010100110101010100, /* arf<1>:f grf:f imm:f */ 0b00000000100000000000, /* arf<1>:ub arf:ub arf:ub */ 0b11010000100101010100, /* arf<1>:f arf:f grf:f */ 0b00101110110011001100, /* grf<1>:d grf:d imm:w */ 0b10110110100011001100, /* grf<1>:d grf:d grf:d */ 0b01010010110101010100, /* grf<1>:f arf:f imm:f */ 0b10010110100001000100, /* grf<1>:ud grf:ud grf:ud */ 0b01010000110101010100, /* arf<1>:f arf:f imm:f */ 0b00110110110011001100, /* grf<1>:d grf:d imm:d */ 0b00010110110001000100, /* grf<1>:ud grf:ud imm:ud */ 0b00000111000101010100, /* grf<2>:f grf:f arf:ub */ 0b00101100110011001100, /* arf<1>:d grf:d imm:w */ 0b00000000100000100010, /* arf<1>:uw arf:uw arf:ub */ 0b00000010100001000100, /* grf<1>:ud arf:ud arf:ub */ 0b00100110110000101010, /* grf<1>:w grf:uw imm:uv */ 0b00001110110000100010, /* grf<1>:uw grf:uw imm:uw */ 0b10010111000001000100, /* grf<2>:ud grf:ud grf:ud */ 0b00000110100101001100, /* grf<1>:d grf:f arf:ub */ 0b10001100100011001100, /* arf<1>:d grf:d grf:uw */ 0b00000110100001010100, /* grf<1>:f grf:ud arf:ub */ 0b00101110110001001100, /* grf<1>:d grf:ud imm:w */ 0b00000010100000100010, /* grf<1>:uw arf:uw arf:ub */ 0b00000110100000110100, /* grf<1>:f grf:uw arf:ub */ 0b00000110100000010100, /* grf<1>:f grf:ub arf:ub */ 0b00000110100011010100, /* grf<1>:f grf:d arf:ub */ 0b00000010100101010100, /* grf<1>:f arf:f arf:ub */ }; static const uint16_t gfx12_subreg_table[32] = { 0b000000000000000, /* .0 .0 .0 */ 0b100000000000000, /* .0 .0 .16 */ 0b001000000000000, /* .0 .0 .4 */ 0b011000000000000, /* .0 .0 .12 */ 0b000000010000000, /* .0 .4 .0 */ 0b010000000000000, /* .0 .0 .8 */ 0b101000000000000, /* .0 .0 .20 */ 0b000000000001000, /* .8 .0 .0 */ 0b000000100000000, /* .0 .8 .0 */ 0b110000000000000, /* .0 .0 .24 */ 0b111000000000000, /* .0 .0 .28 */ 0b000001000000000, /* .0 .16 .0 */ 0b000000000000100, /* .4 .0 .0 */ 0b000001100000000, /* .0 .24 .0 */ 0b000001010000000, /* .0 .20 .0 */ 0b000000110000000, /* .0 .12 .0 */ 0b000001110000000, /* .0 .28 .0 */ 0b000000000011100, /* .28 .0 .0 */ 0b000000000010000, /* .16 .0 .0 */ 0b000000000001100, /* .12 .0 .0 */ 0b000000000011000, /* .24 .0 .0 */ 0b000000000010100, /* .20 .0 .0 */ 0b000000000000010, /* .2 .0 .0 */ 0b000000101000000, /* .0 .10 .0 */ 0b000000001000000, /* .0 .2 .0 */ 0b000000010000100, /* .4 .4 .0 */ 0b000000001011100, /* .28 .2 .0 */ 0b000000001000010, /* .2 .2 .0 */ 0b000000110001100, /* .12 .12 .0 */ 0b000000000100000, /* .0 .1 .0 */ 0b000000001100000, /* .0 .3 .0 */ 0b110001100000000, /* .0 .24 .24 */ }; static const uint16_t gfx12_src0_index_table[16] = { 0b010001100100, /* r<8;8,1> */ 0b000000000000, /* r<0;1,0> */ 0b010001100110, /* -r<8;8,1> */ 0b010001100101, /* (abs)r<8;8,1> */ 0b000000000010, /* -r<0;1,0> */ 0b001000000000, /* r<2;1,0> */ 0b001001000000, /* r<2;4,0> */ 0b001101000000, /* r<4;4,0> */ 0b001000100100, /* r<2;2,1> */ 0b001100000000, /* r<4;1,0> */ 0b001000100110, /* -r<2;2,1> */ 0b001101000100, /* r<4;4,1> */ 0b010001100111, /* -(abs)r<8;8,1> */ 0b000100000000, /* r<1;1,0> */ 0b000000000001, /* (abs)r<0;1,0> */ 0b111100010000, /* r[a]<1,0> */ }; static const uint16_t gfx12_src1_index_table[16] = { 0b000100011001, /* r<8;8,1> */ 0b000000000000, /* r<0;1,0> */ 0b100100011001, /* -r<8;8,1> */ 0b100000000000, /* -r<0;1,0> */ 0b010100011001, /* (abs)r<8;8,1> */ 0b100011010000, /* -r<4;4,0> */ 0b000010000000, /* r<2;1,0> */ 0b000010001001, /* r<2;2,1> */ 0b100010001001, /* -r<2;2,1> */ 0b000011010000, /* r<4;4,0> */ 0b000011010001, /* r<4;4,1> */ 0b000011000000, /* r<4;1,0> */ 0b110100011001, /* -(abs)r<8;8,1> */ 0b010000000000, /* (abs)r<0;1,0> */ 0b110000000000, /* -(abs)r<0;1,0> */ 0b100011010001, /* -r<4;4,1> */ }; static const uint16_t xehp_src0_index_table[16] = { 0b000100000000, /* r<1;1,0> */ 0b000000000000, /* r<0;1,0> */ 0b000100000010, /* -r<1;1,0> */ 0b000100000001, /* (abs)r<1;1,0> */ 0b000000000010, /* -r<0;1,0> */ 0b001000000000, /* r<2;1,0> */ 0b001001000000, /* r<2;4,0> */ 0b001101000000, /* r<4;4,0> */ 0b001100000000, /* r<4;1,0> */ 0b000100000011, /* -(abs)r<1;1,0> */ 0b000000000001, /* (abs)r<0;1,0> */ 0b111100010000, /* r[a]<1,0> */ 0b010001100000, /* r<8;8,0> */ 0b000101000000, /* r<1;4,0> */ 0b010001001000, /* r<8;4,2> */ 0b001000000010, /* -r<2;1,0> */ }; static const uint16_t xehp_src1_index_table[16] = { 0b000001000000, /* r<1;1,0> */ 0b000000000000, /* r<0;1,0> */ 0b100001000000, /* -r<1;1,0> */ 0b100000000000, /* -r<0;1,0> */ 0b010001000000, /* (abs)r<1;1,0> */ 0b100011010000, /* -r<4;4,0> */ 0b000010000000, /* r<2;1,0> */ 0b000011010000, /* r<4;4,0> */ 0b000011000000, /* r<4;1,0> */ 0b110001000000, /* -(abs)r<1;1,0> */ 0b010000000000, /* (abs)r<0;1,0> */ 0b110000000000, /* -(abs)r<0;1,0> */ 0b000100011000, /* r<8;8,0> */ 0b100010000000, /* -r<2;1,0> */ 0b100000001001, /* -r<0;2,1> */ 0b100001000100, /* -r[a]<1;1,0> */ }; static const uint32_t xe2_control_index_table[32] = { 0b000000000000000100, /* (16|M0) */ 0b000000100000000000, /* (W) (1|M0) */ 0b000000000010000100, /* (16|M16) */ 0b000000000000000000, /* (1|M0) */ 0b000000100000000100, /* (W) (16|M0) */ 0b010000000000000100, /* (16|M0) (.ge)f0.0 */ 0b010100000000000100, /* (16|M0) (.lt)f0.0 */ 0b000000100000000010, /* (W) (4|M0) */ 0b000000000000000101, /* (32|M0) */ 0b000000100000000011, /* (W) (8|M0) */ 0b001100100000000000, /* (W) (1|M0) (.gt)f0.0 */ 0b000010000000000100, /* (16|M0) (sat) */ 0b000100000000000100, /* (16|M0) (.eq)f0.0 */ 0b000000100000000001, /* (W) (2|M0) */ 0b001100000000000100, /* (16|M0) (.gt)f0.0 */ 0b000100100000000000, /* (W) (1|M0) (.eq)f0.0 */ 0b010100100000000010, /* (W) (4|M0) (.lt)f0.0 */ 0b010000100000000000, /* (W) (1|M0) (.ge)f0.0 */ 0b010000100000000010, /* (W) (4|M0) (.ge)f0.0 */ 0b010100100000000000, /* (W) (1|M0) (.lt)f0.0 */ 0b001000000000000100, /* (16|M0) (.ne)f0.0 */ 0b000000000100100100, /* (f2.0) (16|M0) */ 0b010100100000000011, /* (W) (8|M0) (.lt)f0.0 */ 0b000000000100011100, /* (f1.1) (16|M0) */ 0b010000100000000011, /* (W) (8|M0) (.ge)f0.0 */ 0b000000000100001100, /* (f0.1) (16|M0) */ 0b000000000100010100, /* (f1.0) (16|M0) */ 0b000000000100110100, /* (f3.0) (16|M0) */ 0b000000000100111100, /* (f3.1) (16|M0) */ 0b000000000100101100, /* (f2.1) (16|M0) */ 0b000000000100000100, /* (f0.0) (16|M0) */ 0b010100000000100100, /* (16|M0) (.lt)f2.0 */ }; static const uint32_t xe2_datatype_table[32] = { 0b11010110100101010100, /* grf<1>:f grf:f grf:f */ 0b11010100100101010100, /* arf<1>:f grf:f grf:f */ 0b00000110100101010100, /* grf<1>:f grf:f arf:ub */ 0b00000110100001000100, /* grf<1>:ud grf:ud arf:ub */ 0b01010110110101010100, /* grf<1>:f grf:f imm:f */ 0b11010010100101010100, /* grf<1>:f arf:f grf:f */ 0b10111110100011101110, /* grf<1>:q grf:q grf:q */ 0b00000000100000000000, /* arf<1>:ub arf:ub arf:ub */ 0b01010110100101010100, /* grf<1>:f grf:f arf:f */ 0b00000010101001000100, /* grf<1>:ud imm:ud */ 0b00101110110011001100, /* grf<1>:d grf:d imm:w */ 0b11010000100101010100, /* arf<1>:f arf:f grf:f */ 0b01010100100101010100, /* arf<1>:f grf:f arf:f */ 0b01010100110101010100, /* arf<1>:f grf:f imm:f */ 0b00000010101101010100, /* grf<1>:f imm:f */ 0b00000110100011001100, /* grf<1>:d grf:d arf:ub */ 0b00101110110011101110, /* grf<1>:q grf:q imm:w */ 0b00000110100001100110, /* grf<1>:uq grf:uq arf:ub */ 0b01010000100101010100, /* arf<1>:f arf:f arf:f */ 0b10110110100011001100, /* grf<1>:d grf:d grf:d */ 0b01010010100101010100, /* grf<1>:f arf:f arf:f */ 0b00000111000001000100, /* grf<2>:ud grf:ud arf:ub */ 0b00110110110011001110, /* grf<1>:q grf:d imm:d */ 0b00101100110011001100, /* arf<1>:d grf:d imm:w */ 0b11011110100101110110, /* grf<1>:df grf:df grf:df */ 0b01010010110101010100, /* grf<1>:f arf:f imm:f */ 0b10010110100001000100, /* grf<1>:ud grf:ud grf:ud */ 0b00000010100001000100, /* grf<1>:ud arf:ud arf:ub */ 0b00001110110001000100, /* grf<1>:ud grf:ud imm:uw */ 0b00000010101010101100, /* grf<1>:d imm:w */ 0b01010000110101010100, /* arf<1>:f arf:f imm:f */ 0b00000100100001000100, /* arf<1>:ud grf:ud arf:ub */ }; static const uint16_t xe2_subreg_table[16] = { 0b000000000000, /* .0 .0 */ 0b000010000000, /* .0 .4 */ 0b000000000100, /* .4 .0 */ 0b010000000000, /* .0 .32 */ 0b001000000000, /* .0 .16 */ 0b000000001000, /* .8 .0 */ 0b000100000000, /* .0 .8 */ 0b010100000000, /* .0 .40 */ 0b011000000000, /* .0 .48 */ 0b000110000000, /* .0 .12 */ 0b000000010000, /* .16 .0 */ 0b011010000000, /* .0 .52 */ 0b001100000000, /* .0 .24 */ 0b011100000000, /* .0 .56 */ 0b010110000000, /* .0 .44 */ 0b010010000000, /* .0 .36 */ }; static const uint16_t xe2_src0_index_table[8] = { 0b00100000000, /* r<1;1,0> */ 0b00000000000, /* r<0;1,0> */ 0b01000000000, /* r<2;1,0> */ 0b00100000010, /* -r<1;1,0> */ 0b01100000000, /* r<4;1,0> */ 0b00100000001, /* (abs)r<1;1,0> */ 0b00000000010, /* -r<0;1,0> */ 0b01001000000, /* r<2;4,0> */ }; static const uint16_t xe2_src1_index_table[16] = { 0b0000100000000000, /* r<1;1,0>.0 */ 0b0000000000000000, /* r<0;1,0>.0 */ 0b1000100000000000, /* -r<1;1,0>.0 */ 0b0000000000010000, /* r<0;1,0>.8 */ 0b0000000000001000, /* r<0;1,0>.4 */ 0b0000000000011000, /* r<0;1,0>.12 */ 0b0000000001010000, /* r<0;1,0>.40 */ 0b0000000001000000, /* r<0;1,0>.32 */ 0b0000000000100000, /* r<0;1,0>.16 */ 0b0000000001111000, /* r<0;1,0>.60 */ 0b0000000000111000, /* r<0;1,0>.28 */ 0b0000000000101000, /* r<0;1,0>.20 */ 0b0000000001011000, /* r<0;1,0>.44 */ 0b0000000001001000, /* r<0;1,0>.36 */ 0b0000000001110000, /* r<0;1,0>.56 */ 0b0000000000110000, /* r<0;1,0>.24 */ }; /* This is actually the control index table for Cherryview (26 bits), but the * only difference from Broadwell (24 bits) is that it has two extra 0-bits at * the start. * * The low 24 bits have the same mappings on both hardware. */ static const uint32_t gfx8_3src_control_index_table[4] = { 0b00100000000110000000000001, 0b00000000000110000000000001, 0b00000000001000000000000001, 0b00000000001000000000100001, }; /* This is actually the control index table for Cherryview (49 bits), but the * only difference from Broadwell (46 bits) is that it has three extra 0-bits * at the start. * * The low 44 bits have the same mappings on both hardware, and since the high * three bits on Broadwell are zero, we can reuse Cherryview's table. */ static const uint64_t gfx8_3src_source_index_table[4] = { 0b0000001110010011100100111001000001111000000000000, 0b0000001110010011100100111001000001111000000000010, 0b0000001110010011100100111001000001111000000001000, 0b0000001110010011100100111001000001111000000100000, }; static const uint64_t gfx12_3src_control_index_table[32] = { 0b000001001010010101000000000000000100, /* (16|M0) grf<1>:f :f :f :f */ 0b000001001010010101000000000000000011, /* (8|M0) grf<1>:f :f :f :f */ 0b000001001000010101000000000000000011, /* (8|M0) arf<1>:f :f :f :f */ 0b000001001010010101000010000000000011, /* (W) (8|M0) grf<1>:f :f :f :f */ 0b000001001000010101000010000000000011, /* (W) (8|M0) arf<1>:f :f :f :f */ 0b000001001000010101000000000000010011, /* (8|M8) arf<1>:f :f :f :f */ 0b000001001010010101000000000000010011, /* (8|M8) grf<1>:f :f :f :f */ 0b000001001000010101000010000000010011, /* (W) (8|M8) arf<1>:f :f :f :f */ 0b000001001010010101000010000000010011, /* (W) (8|M8) grf<1>:f :f :f :f */ 0b000001001010010101000010000000000100, /* (W) (16|M0) grf<1>:f :f :f :f */ 0b000001001000010101000000000000000100, /* (16|M0) arf<1>:f :f :f :f */ 0b000001001010010101010000000000000100, /* (16|M0) (sat)grf<1>:f :f :f :f */ 0b000001001010010101000000000000100100, /* (16|M16) grf<1>:f :f :f :f */ 0b000001001000010101000010000000000100, /* (W) (16|M0) arf<1>:f :f :f :f */ 0b000001001010010101000010000000000000, /* (W) (1|M0) grf<1>:f :f :f :f */ 0b000001001010010101010000000000000011, /* (8|M0) (sat)grf<1>:f :f :f :f */ 0b000001001000010101000010000000110011, /* (W) (8|M24) arf<1>:f :f :f :f */ 0b000001001000010101000010000000100011, /* (W) (8|M16) arf<1>:f :f :f :f */ 0b000001001010010101000010000000110011, /* (W) (8|M24) grf<1>:f :f :f :f */ 0b000001001010010101000010000000100011, /* (W) (8|M16) grf<1>:f :f :f :f */ 0b000001001000010101000000000000100011, /* (8|M16) arf<1>:f :f :f :f */ 0b000001001000010101000000000000110011, /* (8|M24) arf<1>:f :f :f :f */ 0b000001001010010101000000000000100011, /* (8|M16) grf<1>:f :f :f :f */ 0b000001001010010101000000000000110011, /* (8|M24) grf<1>:f :f :f :f */ 0b000001001000010101010000000000000100, /* (16|M0) (sat)arf<1>:f :f :f :f */ 0b000001001010010101010010000000000100, /* (W) (16|M0) (sat)grf<1>:f :f :f :f */ 0b000001001010010101000010000000100100, /* (W) (16|M16) grf<1>:f :f :f :f */ 0b000001001010010001000010000000000000, /* (W) (1|M0) grf<1>:ud :ud :ud :ud */ 0b000001001000010101000000000000100100, /* (16|M16) arf<1>:f :f :f :f */ 0b000001001010010101010000000000100100, /* (16|M16) (sat)grf<1>:f :f :f :f */ 0b000001001010010101000010000000000010, /* (W) (4|M0) grf<1>:f :f :f :f */ 0b000001001000010101010000000000000011, /* (8|M0) (sat)arf<1>:f :f :f :f */ }; static const uint64_t xehp_3src_control_index_table[32] = { 0b0000010010100010101000000000000000100, /* (16|M0) grf<1>:f :f :f :f */ 0b0000010010100010101000000000000000011, /* (8|M0) grf<1>:f :f :f :f */ 0b0000010010000010101000000000000000011, /* (8|M0) arf<1>:f :f :f :f */ 0b0000010010100010101000010000000000011, /* (W) (8|M0) grf<1>:f :f :f :f */ 0b0000010010000010101000010000000000011, /* (W) (8|M0) arf<1>:f :f :f :f */ 0b0000010010000010101000000000000010011, /* (8|M8) arf<1>:f :f :f :f */ 0b0000010010100010101000000000000010011, /* (8|M8) grf<1>:f :f :f :f */ 0b0000010010000010101000010000000010011, /* (W) (8|M8) arf<1>:f :f :f :f */ 0b0000010010100010101000010000000010011, /* (W) (8|M8) grf<1>:f :f :f :f */ 0b0000010010100010101000010000000000100, /* (W) (16|M0) grf<1>:f :f :f :f */ 0b0000010010000010101000000000000000100, /* (16|M0) arf<1>:f :f :f :f */ 0b0000010010100010101010000000000000100, /* (16|M0) (sat)grf<1>:f :f :f :f */ 0b0000010010100010101000000000000100100, /* (16|M16) grf<1>:f :f :f :f */ 0b0000010010000010101000010000000000100, /* (W) (16|M0) arf<1>:f :f :f :f */ 0b0000010010100010101000010000000000000, /* (W) (1|M0) grf<1>:f :f :f :f */ 0b0000010010100010101010000000000000011, /* (8|M0) (sat)grf<1>:f :f :f :f */ 0b0000010010000010101000010000000100011, /* (W) (8|M16) arf<1>:f :f :f :f */ 0b0000010010000010101000010000000110011, /* (W) (8|M24) arf<1>:f :f :f :f */ 0b0000010010100010101000010000000100011, /* (W) (8|M16) grf<1>:f :f :f :f */ 0b0000010010100010101000010000000110011, /* (W) (8|M24) grf<1>:f :f :f :f */ 0b0000010010000010101000000000000110011, /* (8|M24) arf<1>:f :f :f :f */ 0b0000010010000010101000000000000100011, /* (8|M16) arf<1>:f :f :f :f */ 0b0000000100111110011000000000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :ub :b */ 0b0000000000111110011000100000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :ub :ub {Atomic} */ 0b0000100100111110011000100000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :b :b {Atomic} */ 0b0000100000111110011000100000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :b :ub {Atomic} */ 0b0000100100111110011000000000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :b :b */ 0b0000000000111110011000000000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :ub :ub */ 0b0000000100111110011000100000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :ub :b {Atomic} */ 0b0000100000111110011000000000000000011, /* dpas.8x* (8|M0) grf<1>:d :d :b :ub */ 0b0000101101111010101000100000000000011, /* dpas.8x* (8|M0) grf<1>:f :f :bf :bf {Atomic} */ 0b0000101101111010101000000000000000011, /* dpas.8x* (8|M0) grf<1>:f :f :bf :bf */ }; static const uint64_t xe2_3src_control_index_table[16] = { 0b0000010010100010101000000000000100, /* (16|M0) grf<1>:f :f :f :f */ 0b0000010010000010101000000000000100, /* (16|M0) arf<1>:f :f :f :f */ 0b0000010010100010101000100000000100, /* (W)(16|M0) grf<1>:f :f :f :f */ 0b0000010010000010101000100000000100, /* (W)(16|M0) arf<1>:f :f :f :f */ 0b0000011011100011101100000000000100, /* (16|M0) grf<1>:df :df :df :df */ 0b0000011011100011101100000010000100, /* (16|M16) grf<1>:df :df :df :df */ 0b0000011011000011101100000000000100, /* (16|M0) arf<1>:df :df :df :df */ 0b0000010010100010101000000000000101, /* (32|M0) grf<1>:f :f :f :f */ 0b0000010010000010101000000000000101, /* (32|M0) arf<1>:f :f :f :f */ 0b0000010010000010101010000000000100, /* (16|M0) (sat)arf<1>:f :f :f :f */ 0b0000010010100010101010000000000100, /* (16|M0) (sat)grf<1>:f :f :f :f */ 0b0000011011000011101100000010000100, /* (16|M16) arf<1>:df :df :df :df */ 0b0000010010100010101000100000000000, /* (W)(1|M0) grf<1>:f :f :f :f */ 0b0000010010100010001000000000000100, /* (16|M0) grf<1>:ud :ud :ud :ud */ 0b0000110110100110011000000000000101, /* (32|M0) grf<1>:d :d :d :d */ 0b0000011011000011101100000000000011, /* (8|M0) arf<1>:df :df :df :df */ }; static const uint64_t xe2_3src_dpas_control_index_table[16] = { 0b0000000000111110011001000000000100, /* dpas.8x* (16|M0) grf:d :d :ub :ub Atomic */ 0b0000000100111110011001000000000100, /* dpas.8x* (16|M0) grf:d :d :ub :b Atomic */ 0b0000100000111110011001000000000100, /* dpas.8x* (16|M0) grf:d :d :b :ub Atomic */ 0b0000100100111110011001000000000100, /* dpas.8x* (16|M0) grf:d :d :b :b Atomic */ 0b0000000000111110011000000000000100, /* dpas.8x* (16|M0) grf:d :d :ub :ub */ 0b0000100100111110011000000000000100, /* dpas.8x* (16|M0) grf:d :d :b :b */ 0b0000101101111010101001000000000100, /* dpas.8x* (16|M0) grf:f :f :bf :bf Atomic */ 0b0000101101111101101001000000000100, /* dpas.8x* (16|M0) grf:f :bf :bf :bf Atomic */ 0b0000101101111010110101000000000100, /* dpas.8x* (16|M0) grf:bf :f :bf :bf Atomic */ 0b0000101101111101110101000000000100, /* dpas.8x* (16|M0) grf:bf :bf :bf :bf Atomic */ 0b0000101101111010101000000000000100, /* dpas.8x* (16|M0) grf:f :f :bf :bf */ 0b0000001001111010101001000000000100, /* dpas.8x* (16|M0) grf:f :f :hf :hf Atomic */ 0b0000001001111001101001000000000100, /* dpas.8x* (16|M0) grf:f :hf :hf :hf Atomic */ 0b0000001001111010100101000000000100, /* dpas.8x* (16|M0) grf:hf :f :hf :hf Atomic */ 0b0000001001111001100101000000000100, /* dpas.8x* (16|M0) grf:hf :hf :hf :hf Atomic */ 0b0000001001111010101000000000000100, /* dpas.8x* (16|M0) grf:f :f :hf :hf */ }; static const uint32_t gfx12_3src_source_index_table[32] = { 0b100101100001100000000, /* grf<0;0> grf<8;1> grf<0> */ 0b100101100001001000010, /* arf<4;1> grf<8;1> grf<0> */ 0b101101100001101000011, /* grf<8;1> grf<8;1> grf<1> */ 0b100101100001101000011, /* grf<8;1> grf<8;1> grf<0> */ 0b101100000000101000011, /* grf<8;1> grf<0;0> grf<1> */ 0b101101100001101001011, /* -grf<8;1> grf<8;1> grf<1> */ 0b101001100001101000011, /* grf<8;1> arf<8;1> grf<1> */ 0b100001100001100000000, /* grf<0;0> arf<8;1> grf<0> */ 0b101101100001100000000, /* grf<0;0> grf<8;1> grf<1> */ 0b101101100101101000011, /* grf<8;1> grf<8;1> -grf<1> */ 0b101101110001101000011, /* grf<8;1> -grf<8;1> grf<1> */ 0b101100000000100000000, /* grf<0;0> grf<0;0> grf<1> */ 0b100001100001101000011, /* grf<8;1> arf<8;1> grf<0> */ 0b100101110001100000000, /* grf<0;0> -grf<8;1> grf<0> */ 0b100101110001101000011, /* grf<8;1> -grf<8;1> grf<0> */ 0b100101100001101001011, /* -grf<8;1> grf<8;1> grf<0> */ 0b100100000000101000011, /* grf<8;1> grf<0;0> grf<0> */ 0b100101100001100001000, /* -grf<0;0> grf<8;1> grf<0> */ 0b100100000000100000000, /* grf<0;0> grf<0;0> grf<0> */ 0b101101110001100000000, /* grf<0;0> -grf<8;1> grf<1> */ 0b100101100101100000000, /* grf<0;0> grf<8;1> -grf<0> */ 0b101001100001100000000, /* grf<0;0> arf<8;1> grf<1> */ 0b100101100101101000011, /* grf<8;1> grf<8;1> -grf<0> */ 0b101101100101101001011, /* -grf<8;1> grf<8;1> -grf<1> */ 0b101001100001101001011, /* -grf<8;1> arf<8;1> grf<1> */ 0b101101110001101001011, /* -grf<8;1> -grf<8;1> grf<1> */ 0b101100010000101000011, /* grf<8;1> -grf<0;0> grf<1> */ 0b101100000100101000011, /* grf<8;1> grf<0;0> -grf<1> */ 0b101101100001100001000, /* -grf<0;0> grf<8;1> grf<1> */ 0b101101100101100000000, /* grf<0;0> grf<8;1> -grf<1> */ 0b100100000100101000011, /* grf<8;1> grf<0;0> -grf<0> */ 0b101001100101101000011, /* grf<8;1> arf<8;1> -grf<1> */ }; static const uint32_t xehp_3src_source_index_table[32] = { 0b100100000001100000000, /* grf<0;0> grf<1;0> grf<0> */ 0b100100000001000000001, /* arf<1;0> grf<1;0> grf<0> */ 0b101100000001100000001, /* grf<1;0> grf<1;0> grf<1> */ 0b100100000001100000001, /* grf<1;0> grf<1;0> grf<0> */ 0b101100000000100000001, /* grf<1;0> grf<0;0> grf<1> */ 0b101100000001100001001, /* -grf<1;0> grf<1;0> grf<1> */ 0b101000000001100000001, /* grf<1;0> arf<1;0> grf<1> */ 0b101100000001100000000, /* grf<0;0> grf<1;0> grf<1> */ 0b100000000001100000000, /* grf<0;0> arf<1;0> grf<0> */ 0b101100000101100000001, /* grf<1;0> grf<1;0> -grf<1> */ 0b101100010001100000001, /* grf<1;0> -grf<1;0> grf<1> */ 0b101100000000100000000, /* grf<0;0> grf<0;0> grf<1> */ 0b100000000001100000001, /* grf<1;0> arf<1;0> grf<0> */ 0b100100010001100000000, /* grf<0;0> -grf<1;0> grf<0> */ 0b100100010001100000001, /* grf<1;0> -grf<1;0> grf<0> */ 0b100100000001100001001, /* -grf<1;0> grf<1;0> grf<0> */ 0b100100000000100000001, /* grf<1;0> grf<0;0> grf<0> */ 0b100100000001100001000, /* -grf<0;0> grf<1;0> grf<0> */ 0b100100000000100000000, /* grf<0;0> grf<0;0> grf<0> * dpas.*x1 grf:d grf:[ub,b] grf:[ub,b] * dpas.*x1 grf:f grf:bf grf:bf */ 0b101100010001100000000, /* grf<0;0> -grf<1;0> grf<1> */ 0b100100000101100000000, /* grf<0;0> grf<1;0> -grf<0> */ 0b101000000001100000000, /* grf<0;0> arf<1;0> grf<1> */ 0b100100000101100000001, /* grf<1;0> grf<1;0> -grf<0> */ 0b101100000101100001001, /* -grf<1;0> grf<1;0> -grf<1> */ 0b100100010000100000000, /* dpas.*x1 grf:d grf:[u2,s2] grf:[ub,b] */ 0b100100000100100000000, /* dpas.*x1 grf:d grf:[ub,b] grf:[u2,s2] */ 0b100100010100100000000, /* dpas.*x1 grf:d grf:[u2,s2] grf:[u2,s2] */ 0b100100001000100000000, /* dpas.*x1 grf:d grf:[u4,s4] grf:[ub,b] */ 0b100100001100100000000, /* dpas.*x1 grf:d grf:[u4,s4] grf:[u2,s2] */ 0b100100000010100000000, /* dpas.*x1 grf:d grf:[ub,b] grf:[u4,s4] */ 0b100100001010100000000, /* dpas.*x1 grf:d grf:[u4,s4] grf:[u4,s4] */ 0b100100010010100000000, /* dpas.*x1 grf:d grf:[u2,s2] grf:[u4,s4] */ }; static const uint32_t xe2_3src_source_index_table[16] = { 0b101100000001100000001, /* grf<1;0> grf<1;0> grf<1> */ 0b101100000001000000001, /* arf<1;0> grf<1;0> grf<1> */ 0b100100000001100000000, /* grf<0;0> grf<1;0> grf<0> */ 0b100100000001000000001, /* arf<1;0> grf<1;0> grf<0> */ 0b100100000001100000001, /* grf<1;0> grf<1;0> grf<0> */ 0b100000000001100000000, /* grf<0;0> arf<1;0> grf<0> */ 0b100000000001100000001, /* grf<1;0> arf<1;0> grf<0> */ 0b101100000101100000001, /* grf<1;0> grf<1;0> -grf<1> */ 0b101000000001100000001, /* grf<1;0> arf<1;0> grf<1> */ 0b101000000001000000001, /* arf<1;0> arf<1;0> grf<1> */ 0b100000000001000000001, /* arf<1;0> arf<1;0> grf<0> */ 0b100100000000100000000, /* grf<0;0> grf<0;0> grf<0> */ 0b100100000000100000001, /* grf<1;0> grf<0;0> grf<0> */ 0b101100000101000000001, /* arf<1;0> grf<1;0> -grf<1> */ 0b100100010001100000001, /* grf<1;0> -grf<1;0> grf<0> */ 0b100100010001000000001, /* arf<1;0> -grf<1;0> grf<0> */ }; static const uint32_t xe2_3src_dpas_source_index_table[16] = { 0b100100000000100000000, /* dpas.*x1 grf:d grf:[ub,b] grf:[ub,b] * dpas.*x1 grf:[f,bf] grf:bf grf:bf * dpas.*x1 grf:[f,hf] grf:hf grf:hf */ 0b100100000010100000000, /* dpas.*x1 grf:d grf:[ub,b] grf:[u4,s4] */ 0b100100000100100000000, /* dpas.*x1 grf:d grf:[ub,b] grf:[u2,s2] */ 0b100100001000100000000, /* dpas.*x1 grf:d grf:[u4,s4] grf:[ub,b] */ 0b100100001010100000000, /* dpas.*x1 grf:d grf:[u4,s4] grf:[u4,s4] */ 0b100100001100100000000, /* dpas.*x1 grf:d grf:[u4,s4] grf:[u2,s2] */ 0b100100010000100000000, /* dpas.*x1 grf:d grf:[u2,s2] grf:[ub,b] */ 0b100100010010100000000, /* dpas.*x1 grf:d grf:[u2,s2] grf:[u4,s4] */ 0b100100010100100000000, /* dpas.*x1 grf:d grf:[u2,s2] grf:[u2,s2] */ 0b100100000000100000010, /* dpas.*x2 grf:d grf:[ub,b] grf:[ub,b] */ 0b100100000010100000010, /* dpas.*x2 grf:d grf:[ub,b] grf:[u4,s4] */ 0b100100001000100000010, /* dpas.*x2 grf:d grf:[u4,s4] grf:[ub,b] */ 0b100100001010100000010, /* dpas.*x2 grf:d grf:[u4,s4] grf:[u4,s4] */ 0b100100010100100000010, /* dpas.*x2 grf:d grf:[u2,s2] grf:[u2,s2] */ 0b100100000000100001110, /* dpas.*x8 grf:d grf:[ub,b] grf:[ub,b] */ 0b100100001010100001110, /* dpas.*x8 grf:d grf:[u4,s4] grf:[u4,s4] */ }; static const uint32_t gfx12_3src_subreg_table[32] = { 0b00000000000000000000, /* .0 .0 .0 .0 */ 0b00100000000000000000, /* .0 .0 .0 .4 */ 0b00000000000110000000, /* .0 .12 .0 .0 */ 0b10100000000000000000, /* .0 .0 .0 .20 */ 0b10000000001110000000, /* .0 .28 .0 .16 */ 0b01100000000000000000, /* .0 .0 .0 .12 */ 0b01000000000000000000, /* .0 .0 .0 .8 */ 0b00000010000000000000, /* .0 .0 .8 .0 */ 0b00000001000000000000, /* .0 .0 .4 .0 */ 0b11000000000000000000, /* .0 .0 .0 .24 */ 0b10000000000000000000, /* .0 .0 .0 .16 */ 0b11100000000000000000, /* .0 .0 .0 .28 */ 0b00000110000000000000, /* .0 .0 .24 .0 */ 0b00000000000010000000, /* .0 .4 .0 .0 */ 0b00000100000000000000, /* .0 .0 .16 .0 */ 0b00000011000000000000, /* .0 .0 .12 .0 */ 0b00000101000000000000, /* .0 .0 .20 .0 */ 0b00000111000000000000, /* .0 .0 .28 .0 */ 0b00000000000100000000, /* .0 .8 .0 .0 */ 0b00000000001000000000, /* .0 .16 .0 .0 */ 0b00000000001100000000, /* .0 .24 .0 .0 */ 0b00000000001010000000, /* .0 .20 .0 .0 */ 0b00000000001110000000, /* .0 .28 .0 .0 */ 0b11000000001110000000, /* .0 .28 .0 .24 */ 0b00100000000100000000, /* .0 .8 .0 .4 */ 0b00100000000110000000, /* .0 .12 .0 .4 */ 0b01000000000110000000, /* .0 .12 .0 .8 */ 0b10000000001100000000, /* .0 .24 .0 .16 */ 0b10000000001010000000, /* .0 .20 .0 .16 */ 0b01100000000010000000, /* .0 .4 .0 .12 */ 0b10100000001110000000, /* .0 .28 .0 .20 */ 0b01000000000010000000, /* .0 .4 .0 .8 */ }; static const uint32_t xe2_3src_subreg_table[32] = { 0b00000000000000000000, /* .0 .0 .0 .0 */ 0b00100000000000000000, /* .0 .0 .0 .8 */ 0b10000000000000000000, /* .0 .0 .0 .32 */ 0b00010000000000000000, /* .0 .0 .0 .4 */ 0b11100000000000000000, /* .0 .0 .0 .56 */ 0b01010000000000000000, /* .0 .0 .0 .20 */ 0b10110000000000000000, /* .0 .0 .0 .44 */ 0b01000000000011000000, /* .0 .12 .0 .16 */ 0b01100000000000000000, /* .0 .0 .0 .24 */ 0b10100000000000000000, /* .0 .0 .0 .40 */ 0b11000000000000000000, /* .0 .0 .0 .48 */ 0b01000000000000000000, /* .0 .0 .0 .16 */ 0b01110000000110000000, /* .0 .24 .0 .28 */ 0b10100000001001000000, /* .0 .36 .0 .40 */ 0b11010000001100000000, /* .0 .48 .0 .52 */ 0b01110000000000000000, /* .0 .0 .0 .28 */ 0b11110000000000000000, /* .0 .0 .0 .60 */ 0b10010000000000000000, /* .0 .0 .0 .36 */ 0b00110000000000000000, /* .0 .0 .0 .12 */ 0b00100000000010000000, /* .0 .8 .0 .8 */ 0b00010000000001000000, /* .0 .4 .0 .4 */ 0b00110000000011000000, /* .0 .12 .0 .12 */ 0b11010000000000000000, /* .0 .0 .0 .52 */ 0b00000000000001000000, /* .0 .4 .0 .0 */ 0b00000101100000000000, /* .0 .0 .44 .0 */ 0b00000100000000000000, /* .0 .0 .32 .0 */ 0b00000000000010000000, /* .0 .8 .0 .0 */ 0b00000000001100000000, /* .0 .48 .0 .0 */ 0b00000000001101000000, /* .0 .52 .0 .0 */ 0b00000110100000000000, /* .0 .0 .52 .0 */ 0b00000000001000000000, /* .0 .32 .0 .0 */ 0b00000000001111000000, /* .0 .60 .0 .0 */ }; struct compaction_state { const struct brw_isa_info *isa; const uint32_t *control_index_table; const uint32_t *datatype_table; const uint16_t *subreg_table; const uint16_t *src0_index_table; const uint16_t *src1_index_table; }; static void compaction_state_init(struct compaction_state *c, const struct brw_isa_info *isa); static bool set_control_index(const struct compaction_state *c, brw_eu_compact_inst *dst, const brw_eu_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; uint32_t uncompacted; /* 19b/IVB+; 21b/TGL+ */ if (devinfo->ver >= 20) { uncompacted = (brw_eu_inst_bits(src, 95, 92) << 14) | /* 4b */ (brw_eu_inst_bits(src, 34, 34) << 13) | /* 1b */ (brw_eu_inst_bits(src, 32, 32) << 12) | /* 1b */ (brw_eu_inst_bits(src, 31, 31) << 11) | /* 1b */ (brw_eu_inst_bits(src, 28, 28) << 10) | /* 1b */ (brw_eu_inst_bits(src, 27, 26) << 8) | /* 2b */ (brw_eu_inst_bits(src, 25, 24) << 6) | /* 2b */ (brw_eu_inst_bits(src, 23, 21) << 3) | /* 3b */ (brw_eu_inst_bits(src, 20, 18)); /* 3b */ } else if (devinfo->ver >= 12) { uncompacted = (brw_eu_inst_bits(src, 95, 92) << 17) | /* 4b */ (brw_eu_inst_bits(src, 34, 34) << 16) | /* 1b */ (brw_eu_inst_bits(src, 33, 33) << 15) | /* 1b */ (brw_eu_inst_bits(src, 32, 32) << 14) | /* 1b */ (brw_eu_inst_bits(src, 31, 31) << 13) | /* 1b */ (brw_eu_inst_bits(src, 28, 28) << 12) | /* 1b */ (brw_eu_inst_bits(src, 27, 24) << 8) | /* 4b */ (brw_eu_inst_bits(src, 23, 22) << 6) | /* 2b */ (brw_eu_inst_bits(src, 21, 19) << 3) | /* 3b */ (brw_eu_inst_bits(src, 18, 16)); /* 3b */ } else { uncompacted = (brw_eu_inst_bits(src, 33, 31) << 16) | /* 3b */ (brw_eu_inst_bits(src, 23, 12) << 4) | /* 12b */ (brw_eu_inst_bits(src, 10, 9) << 2) | /* 2b */ (brw_eu_inst_bits(src, 34, 34) << 1) | /* 1b */ (brw_eu_inst_bits(src, 8, 8)); /* 1b */ } for (int i = 0; i < 32; i++) { if (c->control_index_table[i] == uncompacted) { brw_eu_compact_inst_set_control_index(devinfo, dst, i); return true; } } return false; } static bool set_datatype_index(const struct compaction_state *c, brw_eu_compact_inst *dst, const brw_eu_inst *src, bool is_immediate) { const struct intel_device_info *devinfo = c->isa->devinfo; uint32_t uncompacted; /* 18b/G45+; 21b/BDW+; 20b/TGL+ */ if (devinfo->ver >= 12) { uncompacted = (brw_eu_inst_bits(src, 91, 88) << 15) | /* 4b */ (brw_eu_inst_bits(src, 66, 66) << 14) | /* 1b */ (brw_eu_inst_bits(src, 50, 50) << 13) | /* 1b */ (brw_eu_inst_bits(src, 49, 48) << 11) | /* 2b */ (brw_eu_inst_bits(src, 47, 47) << 10) | /* 1b */ (brw_eu_inst_bits(src, 46, 46) << 9) | /* 1b */ (brw_eu_inst_bits(src, 43, 40) << 5) | /* 4b */ (brw_eu_inst_bits(src, 39, 36) << 1) | /* 4b */ (brw_eu_inst_bits(src, 35, 35)); /* 1b */ /* Src1.RegFile overlaps with the immediate, so ignore it if an immediate * is present */ if (!is_immediate) { uncompacted |= brw_eu_inst_bits(src, 98, 98) << 19; /* 1b */ } } else { uncompacted = (brw_eu_inst_bits(src, 63, 61) << 18) | /* 3b */ (brw_eu_inst_bits(src, 94, 89) << 12) | /* 6b */ (brw_eu_inst_bits(src, 46, 35)); /* 12b */ } for (int i = 0; i < 32; i++) { if (c->datatype_table[i] == uncompacted) { brw_eu_compact_inst_set_datatype_index(devinfo, dst, i); return true; } } return false; } static bool set_subreg_index(const struct compaction_state *c, brw_eu_compact_inst *dst, const brw_eu_inst *src, bool is_immediate) { const struct intel_device_info *devinfo = c->isa->devinfo; const unsigned table_len = devinfo->ver >= 20 ? ARRAY_SIZE(xe2_subreg_table) : ARRAY_SIZE(g45_subreg_table); uint16_t uncompacted; /* 15b/G45+; 12b/Xe2+ */ if (devinfo->ver >= 20) { uncompacted = (brw_eu_inst_bits(src, 33, 33) << 0) | /* 1b */ (brw_eu_inst_bits(src, 55, 51) << 1) | /* 5b */ (brw_eu_inst_bits(src, 71, 67) << 6) | /* 5b */ (brw_eu_inst_bits(src, 87, 87) << 11); /* 1b */ } else if (devinfo->ver >= 12) { uncompacted = (brw_eu_inst_bits(src, 55, 51) << 0) | /* 5b */ (brw_eu_inst_bits(src, 71, 67) << 5); /* 5b */ if (!is_immediate) uncompacted |= brw_eu_inst_bits(src, 103, 99) << 10; /* 5b */ } else { uncompacted = (brw_eu_inst_bits(src, 52, 48) << 0) | /* 5b */ (brw_eu_inst_bits(src, 68, 64) << 5); /* 5b */ if (!is_immediate) uncompacted |= brw_eu_inst_bits(src, 100, 96) << 10; /* 5b */ } for (int i = 0; i < table_len; i++) { if (c->subreg_table[i] == uncompacted) { brw_eu_compact_inst_set_subreg_index(devinfo, dst, i); return true; } } return false; } static bool set_src0_index(const struct compaction_state *c, brw_eu_compact_inst *dst, const brw_eu_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; uint16_t uncompacted; /* 12b/G45+; 11b/Xe2+ */ int table_len; if (devinfo->ver >= 12) { table_len = (devinfo->ver >= 20 ? ARRAY_SIZE(xe2_src0_index_table) : ARRAY_SIZE(gfx12_src0_index_table)); uncompacted = (devinfo->ver >= 20 ? 0 : brw_eu_inst_bits(src, 87, 87) << 11) | /* 1b */ (brw_eu_inst_bits(src, 86, 84) << 8) | /* 3b */ (brw_eu_inst_bits(src, 83, 81) << 5) | /* 3b */ (brw_eu_inst_bits(src, 80, 80) << 4) | /* 1b */ (brw_eu_inst_bits(src, 65, 64) << 2) | /* 2b */ (brw_eu_inst_bits(src, 45, 44)); /* 2b */ } else { table_len = ARRAY_SIZE(gfx8_src_index_table); uncompacted = brw_eu_inst_bits(src, 88, 77); /* 12b */ } for (int i = 0; i < table_len; i++) { if (c->src0_index_table[i] == uncompacted) { brw_eu_compact_inst_set_src0_index(devinfo, dst, i); return true; } } return false; } static bool set_src1_index(const struct compaction_state *c, brw_eu_compact_inst *dst, const brw_eu_inst *src, bool is_immediate, unsigned imm) { const struct intel_device_info *devinfo = c->isa->devinfo; if (is_immediate) { if (devinfo->ver >= 12) { /* src1 index takes the low 4 bits of the 12-bit compacted value */ brw_eu_compact_inst_set_src1_index(devinfo, dst, imm & 0xf); } else { /* src1 index takes the high 5 bits of the 13-bit compacted value */ brw_eu_compact_inst_set_src1_index(devinfo, dst, imm >> 8); } return true; } else { uint16_t uncompacted; /* 12b/G45+ 16b/Xe2+ */ int table_len; if (devinfo->ver >= 20) { table_len = ARRAY_SIZE(xe2_src1_index_table); uncompacted = (brw_eu_inst_bits(src, 121, 120) << 14) | /* 2b */ (brw_eu_inst_bits(src, 118, 116) << 11) | /* 3b */ (brw_eu_inst_bits(src, 115, 113) << 8) | /* 3b */ (brw_eu_inst_bits(src, 112, 112) << 7) | /* 1b */ (brw_eu_inst_bits(src, 103, 99) << 2) | /* 5b */ (brw_eu_inst_bits(src, 97, 96)); /* 2b */ } else if (devinfo->ver >= 12) { table_len = ARRAY_SIZE(gfx12_src0_index_table); uncompacted = (brw_eu_inst_bits(src, 121, 120) << 10) | /* 2b */ (brw_eu_inst_bits(src, 119, 116) << 6) | /* 4b */ (brw_eu_inst_bits(src, 115, 113) << 3) | /* 3b */ (brw_eu_inst_bits(src, 112, 112) << 2) | /* 1b */ (brw_eu_inst_bits(src, 97, 96)); /* 2b */ } else { table_len = ARRAY_SIZE(gfx8_src_index_table); uncompacted = brw_eu_inst_bits(src, 120, 109); /* 12b */ } for (int i = 0; i < table_len; i++) { if (c->src1_index_table[i] == uncompacted) { brw_eu_compact_inst_set_src1_index(devinfo, dst, i); return true; } } } return false; } static bool set_3src_control_index(const struct intel_device_info *devinfo, brw_eu_compact_inst *dst, const brw_eu_inst *src, bool is_dpas) { if (devinfo->ver >= 20) { assert(is_dpas || !brw_eu_inst_bits(src, 49, 49)); const uint64_t uncompacted = /* 34b/Xe2+ */ (brw_eu_inst_bits(src, 95, 92) << 30) | /* 4b */ (brw_eu_inst_bits(src, 90, 88) << 27) | /* 3b */ (brw_eu_inst_bits(src, 82, 80) << 24) | /* 3b */ (brw_eu_inst_bits(src, 50, 50) << 23) | /* 1b */ (brw_eu_inst_bits(src, 49, 48) << 21) | /* 2b */ (brw_eu_inst_bits(src, 42, 40) << 18) | /* 3b */ (brw_eu_inst_bits(src, 39, 39) << 17) | /* 1b */ (brw_eu_inst_bits(src, 38, 36) << 14) | /* 3b */ (brw_eu_inst_bits(src, 34, 34) << 13) | /* 1b */ (brw_eu_inst_bits(src, 32, 32) << 12) | /* 1b */ (brw_eu_inst_bits(src, 31, 31) << 11) | /* 1b */ (brw_eu_inst_bits(src, 28, 28) << 10) | /* 1b */ (brw_eu_inst_bits(src, 27, 26) << 8) | /* 2b */ (brw_eu_inst_bits(src, 25, 24) << 6) | /* 2b */ (brw_eu_inst_bits(src, 23, 21) << 3) | /* 3b */ (brw_eu_inst_bits(src, 20, 18)); /* 3b */ /* The bits used to index the tables for 3src and 3src-dpas * are the same, so just need to pick the right one. */ const uint64_t *table = is_dpas ? xe2_3src_dpas_control_index_table : xe2_3src_control_index_table; const unsigned size = is_dpas ? ARRAY_SIZE(xe2_3src_dpas_control_index_table) : ARRAY_SIZE(xe2_3src_control_index_table); for (unsigned i = 0; i < size; i++) { if (table[i] == uncompacted) { brw_eu_compact_inst_set_3src_control_index(devinfo, dst, i); return true; } } } else if (devinfo->verx10 >= 125) { uint64_t uncompacted = /* 37b/XeHP+ */ (brw_eu_inst_bits(src, 95, 92) << 33) | /* 4b */ (brw_eu_inst_bits(src, 90, 88) << 30) | /* 3b */ (brw_eu_inst_bits(src, 82, 80) << 27) | /* 3b */ (brw_eu_inst_bits(src, 50, 50) << 26) | /* 1b */ (brw_eu_inst_bits(src, 49, 48) << 24) | /* 2b */ (brw_eu_inst_bits(src, 42, 40) << 21) | /* 3b */ (brw_eu_inst_bits(src, 39, 39) << 20) | /* 1b */ (brw_eu_inst_bits(src, 38, 36) << 17) | /* 3b */ (brw_eu_inst_bits(src, 34, 34) << 16) | /* 1b */ (brw_eu_inst_bits(src, 33, 33) << 15) | /* 1b */ (brw_eu_inst_bits(src, 32, 32) << 14) | /* 1b */ (brw_eu_inst_bits(src, 31, 31) << 13) | /* 1b */ (brw_eu_inst_bits(src, 28, 28) << 12) | /* 1b */ (brw_eu_inst_bits(src, 27, 24) << 8) | /* 4b */ (brw_eu_inst_bits(src, 23, 23) << 7) | /* 1b */ (brw_eu_inst_bits(src, 22, 22) << 6) | /* 1b */ (brw_eu_inst_bits(src, 21, 19) << 3) | /* 3b */ (brw_eu_inst_bits(src, 18, 16)); /* 3b */ for (unsigned i = 0; i < ARRAY_SIZE(xehp_3src_control_index_table); i++) { if (xehp_3src_control_index_table[i] == uncompacted) { brw_eu_compact_inst_set_3src_control_index(devinfo, dst, i); return true; } } } else if (devinfo->ver >= 12) { uint64_t uncompacted = /* 36b/TGL+ */ (brw_eu_inst_bits(src, 95, 92) << 32) | /* 4b */ (brw_eu_inst_bits(src, 90, 88) << 29) | /* 3b */ (brw_eu_inst_bits(src, 82, 80) << 26) | /* 3b */ (brw_eu_inst_bits(src, 50, 50) << 25) | /* 1b */ (brw_eu_inst_bits(src, 48, 48) << 24) | /* 1b */ (brw_eu_inst_bits(src, 42, 40) << 21) | /* 3b */ (brw_eu_inst_bits(src, 39, 39) << 20) | /* 1b */ (brw_eu_inst_bits(src, 38, 36) << 17) | /* 3b */ (brw_eu_inst_bits(src, 34, 34) << 16) | /* 1b */ (brw_eu_inst_bits(src, 33, 33) << 15) | /* 1b */ (brw_eu_inst_bits(src, 32, 32) << 14) | /* 1b */ (brw_eu_inst_bits(src, 31, 31) << 13) | /* 1b */ (brw_eu_inst_bits(src, 28, 28) << 12) | /* 1b */ (brw_eu_inst_bits(src, 27, 24) << 8) | /* 4b */ (brw_eu_inst_bits(src, 23, 23) << 7) | /* 1b */ (brw_eu_inst_bits(src, 22, 22) << 6) | /* 1b */ (brw_eu_inst_bits(src, 21, 19) << 3) | /* 3b */ (brw_eu_inst_bits(src, 18, 16)); /* 3b */ for (unsigned i = 0; i < ARRAY_SIZE(gfx12_3src_control_index_table); i++) { if (gfx12_3src_control_index_table[i] == uncompacted) { brw_eu_compact_inst_set_3src_control_index(devinfo, dst, i); return true; } } } else { uint32_t uncompacted = /* 26b/SKL+ */ (brw_eu_inst_bits(src, 36, 35) << 24) | /* 2b */ (brw_eu_inst_bits(src, 34, 32) << 21) | /* 3b */ (brw_eu_inst_bits(src, 28, 8)); /* 21b */ for (unsigned i = 0; i < ARRAY_SIZE(gfx8_3src_control_index_table); i++) { if (gfx8_3src_control_index_table[i] == uncompacted) { brw_eu_compact_inst_set_3src_control_index(devinfo, dst, i); return true; } } } return false; } static bool set_3src_source_index(const struct intel_device_info *devinfo, brw_eu_compact_inst *dst, const brw_eu_inst *src, bool is_dpas) { if (devinfo->ver >= 12) { uint32_t uncompacted = /* 21b/TGL+ */ (brw_eu_inst_bits(src, 114, 114) << 20) | /* 1b */ (brw_eu_inst_bits(src, 113, 112) << 18) | /* 2b */ (brw_eu_inst_bits(src, 98, 98) << 17) | /* 1b */ (brw_eu_inst_bits(src, 97, 96) << 15) | /* 2b */ (brw_eu_inst_bits(src, 91, 91) << 14) | /* 1b */ (brw_eu_inst_bits(src, 87, 86) << 12) | /* 2b */ (brw_eu_inst_bits(src, 85, 84) << 10) | /* 2b */ (brw_eu_inst_bits(src, 83, 83) << 9) | /* 1b */ (brw_eu_inst_bits(src, 66, 66) << 8) | /* 1b */ (brw_eu_inst_bits(src, 65, 64) << 6) | /* 2b */ (brw_eu_inst_bits(src, 47, 47) << 5) | /* 1b */ (brw_eu_inst_bits(src, 46, 46) << 4) | /* 1b */ (brw_eu_inst_bits(src, 45, 44) << 2) | /* 2b */ (brw_eu_inst_bits(src, 43, 43) << 1) | /* 1b */ (brw_eu_inst_bits(src, 35, 35)); /* 1b */ /* In Xe2, the bits used to index the tables for 3src and 3src-dpas * are the same, so just need to pick the right one. */ const uint32_t *three_src_source_index_table = devinfo->ver >= 20 ? (is_dpas ? xe2_3src_dpas_source_index_table : xe2_3src_source_index_table) : devinfo->verx10 >= 125 ? xehp_3src_source_index_table : gfx12_3src_source_index_table; const uint32_t three_src_source_index_table_len = devinfo->ver >= 20 ? (is_dpas ? ARRAY_SIZE(xe2_3src_dpas_source_index_table) : ARRAY_SIZE(xe2_3src_source_index_table)) : devinfo->verx10 >= 125 ? ARRAY_SIZE(xehp_3src_source_index_table) : ARRAY_SIZE(gfx12_3src_source_index_table); for (unsigned i = 0; i < three_src_source_index_table_len; i++) { if (three_src_source_index_table[i] == uncompacted) { brw_eu_compact_inst_set_3src_source_index(devinfo, dst, i); return true; } } } else { uint64_t uncompacted = /* 49b/SKL+ */ (brw_eu_inst_bits(src, 126, 125) << 47) | /* 2b */ (brw_eu_inst_bits(src, 105, 104) << 45) | /* 2b */ (brw_eu_inst_bits(src, 84, 84) << 44) | /* 1b */ (brw_eu_inst_bits(src, 83, 83) << 43) | /* 1b */ (brw_eu_inst_bits(src, 114, 107) << 35) | /* 8b */ (brw_eu_inst_bits(src, 93, 86) << 27) | /* 8b */ (brw_eu_inst_bits(src, 72, 65) << 19) | /* 8b */ (brw_eu_inst_bits(src, 55, 37)); /* 19b */ for (unsigned i = 0; i < ARRAY_SIZE(gfx8_3src_source_index_table); i++) { if (gfx8_3src_source_index_table[i] == uncompacted) { brw_eu_compact_inst_set_3src_source_index(devinfo, dst, i); return true; } } } return false; } static bool set_3src_subreg_index(const struct intel_device_info *devinfo, brw_eu_compact_inst *dst, const brw_eu_inst *src) { assert(devinfo->ver >= 12); uint32_t uncompacted = /* 20b/TGL+ */ (brw_eu_inst_bits(src, 119, 115) << 15) | /* 5b */ (brw_eu_inst_bits(src, 103, 99) << 10) | /* 5b */ (brw_eu_inst_bits(src, 71, 67) << 5) | /* 5b */ (brw_eu_inst_bits(src, 55, 51)); /* 5b */ const uint32_t *table = devinfo->ver >= 20 ? xe2_3src_subreg_table : gfx12_3src_subreg_table; const uint32_t len = devinfo->ver >= 20 ? ARRAY_SIZE(xe2_3src_subreg_table) : ARRAY_SIZE(gfx12_3src_subreg_table); for (unsigned i = 0; i < len; i++) { if (table[i] == uncompacted) { brw_eu_compact_inst_set_3src_subreg_index(devinfo, dst, i); return true; } } return false; } static bool has_unmapped_bits(const struct brw_isa_info *isa, const brw_eu_inst *src) { const struct intel_device_info *devinfo = isa->devinfo; /* EOT can only be mapped on a send if the src1 is an immediate */ if ((brw_eu_inst_opcode(isa, src) == BRW_OPCODE_SENDC || brw_eu_inst_opcode(isa, src) == BRW_OPCODE_SEND) && brw_eu_inst_eot(devinfo, src)) return true; /* Check for instruction bits that don't map to any of the fields of the * compacted instruction. The instruction cannot be compacted if any of * them are set. They overlap with: * - NibCtrl (bit 11 on Gfx8) * - Dst.AddrImm[9] (bit 47 on Gfx8) * - Src0.AddrImm[9] (bit 95 on Gfx8) * - Imm64[27:31] (bit 95 on Gfx8) * - UIP[31] (bit 95 on Gfx8) */ if (devinfo->ver >= 12) { assert(!brw_eu_inst_bits(src, 7, 7)); return false; } else { assert(!brw_eu_inst_bits(src, 7, 7)); return brw_eu_inst_bits(src, 95, 95) || brw_eu_inst_bits(src, 47, 47) || brw_eu_inst_bits(src, 11, 11); } } static bool has_3src_unmapped_bits(const struct intel_device_info *devinfo, const brw_eu_inst *src, bool is_dpas) { /* Check for three-source instruction bits that don't map to any of the * fields of the compacted instruction. All of them seem to be reserved * bits currently. */ if (devinfo->ver >= 20) { assert(is_dpas || !brw_eu_inst_bits(src, 49, 49)); assert(!brw_eu_inst_bits(src, 33, 33)); assert(!brw_eu_inst_bits(src, 7, 7)); } else if (devinfo->ver >= 12) { assert(is_dpas || !brw_eu_inst_bits(src, 49, 49)); assert(!brw_eu_inst_bits(src, 7, 7)); } else { assert(!brw_eu_inst_bits(src, 127, 127) && !brw_eu_inst_bits(src, 7, 7)); } return false; } static bool brw_try_compact_3src_instruction(const struct brw_isa_info *isa, brw_eu_compact_inst *dst, const brw_eu_inst *src) { const struct intel_device_info *devinfo = isa->devinfo; bool is_dpas = brw_eu_inst_opcode(isa, src) == BRW_OPCODE_DPAS; if (has_3src_unmapped_bits(devinfo, src, is_dpas)) return false; #define compact(field) \ brw_eu_compact_inst_set_3src_##field(devinfo, dst, brw_eu_inst_3src_##field(devinfo, src)) #define compact_a16(field) \ brw_eu_compact_inst_set_3src_##field(devinfo, dst, brw_eu_inst_3src_a16_##field(devinfo, src)) compact(hw_opcode); if (!set_3src_control_index(devinfo, dst, src, is_dpas)) return false; if (!set_3src_source_index(devinfo, dst, src, is_dpas)) return false; if (devinfo->ver >= 12) { if (!set_3src_subreg_index(devinfo, dst, src)) return false; compact(swsb); compact(debug_control); compact(dst_reg_nr); compact(src0_reg_nr); compact(src1_reg_nr); compact(src2_reg_nr); } else { compact(dst_reg_nr); compact_a16(src0_rep_ctrl); compact(debug_control); compact(saturate); compact_a16(src1_rep_ctrl); compact_a16(src2_rep_ctrl); compact(src0_reg_nr); compact(src1_reg_nr); compact(src2_reg_nr); compact_a16(src0_subreg_nr); compact_a16(src1_subreg_nr); compact_a16(src2_subreg_nr); } brw_eu_compact_inst_set_3src_cmpt_control(devinfo, dst, true); #undef compact #undef compact_a16 return true; } /* On SNB through ICL, compacted instructions have 12-bits for immediate * sources, and a 13th bit that's replicated through the high 20 bits. * * Effectively this means we get 12-bit integers, 0.0f, and some limited uses * of packed vectors as compactable immediates. * * On TGL+, the high 12-bits of floating-point values (:f and :hf) are encoded * rather than the low 12-bits. For signed integer the 12th bit is replicated, * while for unsigned integers it is not. * * Returns the compacted immediate, or -1 if immediate cannot be compacted */ static int compact_immediate(const struct intel_device_info *devinfo, enum brw_reg_type type, unsigned imm) { if (devinfo->ver >= 12) { /* 16-bit immediates need to be replicated through the 32-bit immediate * field */ switch (type) { case BRW_TYPE_W: case BRW_TYPE_UW: case BRW_TYPE_HF: if ((imm >> 16) != (imm & 0xffff)) return -1; break; default: break; } switch (type) { case BRW_TYPE_F: /* We get the high 12-bits as-is; rest must be zero */ if ((imm & 0xfffff) == 0) return (imm >> 20) & 0xfff; break; case BRW_TYPE_HF: /* We get the high 12-bits as-is; rest must be zero */ if ((imm & 0xf) == 0) return (imm >> 4) & 0xfff; break; case BRW_TYPE_UD: case BRW_TYPE_VF: case BRW_TYPE_UV: case BRW_TYPE_V: /* We get the low 12-bits as-is; rest must be zero */ if ((imm & 0xfffff000) == 0) return imm & 0xfff; break; case BRW_TYPE_UW: /* We get the low 12-bits as-is; rest must be zero */ if ((imm & 0xf000) == 0) return imm & 0xfff; break; case BRW_TYPE_D: /* We get the low 11-bits as-is; 12th is replicated */ if (((int)imm >> 11) == 0 || ((int)imm >> 11) == -1) return imm & 0xfff; break; case BRW_TYPE_W: /* We get the low 11-bits as-is; 12th is replicated */ if (((short)imm >> 11) == 0 || ((short)imm >> 11) == -1) return imm & 0xfff; break; case BRW_TYPE_DF: case BRW_TYPE_Q: case BRW_TYPE_UQ: case BRW_TYPE_B: case BRW_TYPE_UB: default: return -1; } } else { /* We get the low 12 bits as-is; 13th is replicated */ if (((int)imm >> 12) == 0 || ((int)imm >> 12 == -1)) { return imm & 0x1fff; } } return -1; } static int uncompact_immediate(const struct intel_device_info *devinfo, enum brw_reg_type type, unsigned compact_imm) { if (devinfo->ver >= 12) { switch (type) { case BRW_TYPE_F: return compact_imm << 20; case BRW_TYPE_HF: return (compact_imm << 20) | (compact_imm << 4); case BRW_TYPE_UD: case BRW_TYPE_VF: case BRW_TYPE_UV: case BRW_TYPE_V: return compact_imm; case BRW_TYPE_UW: /* Replicate */ return compact_imm << 16 | compact_imm; case BRW_TYPE_D: /* Extend the 12th bit into the high 20 bits */ return (int)(compact_imm << 20) >> 20; case BRW_TYPE_W: /* Extend the 12th bit into the high 4 bits and replicate */ return ((int)(compact_imm << 20) >> 4) | ((unsigned short)((short)(compact_imm << 4) >> 4)); case BRW_TYPE_DF: case BRW_TYPE_Q: case BRW_TYPE_UQ: case BRW_TYPE_B: case BRW_TYPE_UB: unreachable("not reached"); default: unreachable("invalid type"); } } else { /* Replicate the 13th bit into the high 19 bits */ return (int)(compact_imm << 19) >> 19; } unreachable("not reached"); } static bool has_immediate(const struct intel_device_info *devinfo, const brw_eu_inst *inst, enum brw_reg_type *type) { if (brw_eu_inst_src0_reg_file(devinfo, inst) == IMM) { *type = brw_eu_inst_src0_type(devinfo, inst); return *type != BRW_TYPE_INVALID; } else if (brw_eu_inst_src1_reg_file(devinfo, inst) == IMM) { *type = brw_eu_inst_src1_type(devinfo, inst); return *type != BRW_TYPE_INVALID; } return false; } /** * Applies some small changes to instruction types to increase chances of * compaction. */ static brw_eu_inst precompact(const struct brw_isa_info *isa, brw_eu_inst inst) { const struct intel_device_info *devinfo = isa->devinfo; /* In XeHP the compaction tables removed the entries for source regions * <8;8,1> giving preference to <1;1,0> as the way to indicate * sequential elements, so convert to those before compacting. */ if (devinfo->verx10 >= 125) { if (brw_eu_inst_src0_reg_file(devinfo, &inst) == FIXED_GRF && brw_eu_inst_src0_vstride(devinfo, &inst) > BRW_VERTICAL_STRIDE_1 && brw_eu_inst_src0_vstride(devinfo, &inst) == (brw_eu_inst_src0_width(devinfo, &inst) + 1) && brw_eu_inst_src0_hstride(devinfo, &inst) == BRW_HORIZONTAL_STRIDE_1) { brw_eu_inst_set_src0_vstride(devinfo, &inst, BRW_VERTICAL_STRIDE_1); brw_eu_inst_set_src0_width(devinfo, &inst, BRW_WIDTH_1); brw_eu_inst_set_src0_hstride(devinfo, &inst, BRW_HORIZONTAL_STRIDE_0); } if (brw_eu_inst_src1_reg_file(devinfo, &inst) == FIXED_GRF && brw_eu_inst_src1_vstride(devinfo, &inst) > BRW_VERTICAL_STRIDE_1 && brw_eu_inst_src1_vstride(devinfo, &inst) == (brw_eu_inst_src1_width(devinfo, &inst) + 1) && brw_eu_inst_src1_hstride(devinfo, &inst) == BRW_HORIZONTAL_STRIDE_1) { brw_eu_inst_set_src1_vstride(devinfo, &inst, BRW_VERTICAL_STRIDE_1); brw_eu_inst_set_src1_width(devinfo, &inst, BRW_WIDTH_1); brw_eu_inst_set_src1_hstride(devinfo, &inst, BRW_HORIZONTAL_STRIDE_0); } } if (brw_eu_inst_src0_reg_file(devinfo, &inst) != IMM) return inst; /* The Bspec's section titled "Non-present Operands" claims that if src0 * is an immediate that src1's type must be the same as that of src0. * * The SNB+ DataTypeIndex instruction compaction tables contain mappings * that do not follow this rule. E.g., from the IVB/HSW table: * * DataTypeIndex 18-Bit Mapping Mapped Meaning * 3 001000001011111101 r:f | i:vf | a:ud | <1> | dir | * * And from the SNB table: * * DataTypeIndex 18-Bit Mapping Mapped Meaning * 8 001000000111101100 a:w | i:w | a:ud | <1> | dir | * * Neither of these cause warnings from the simulator when used, * compacted or otherwise. In fact, all compaction mappings that have an * immediate in src0 use a:ud for src1. * * Don't do any of this for 64-bit immediates, since the src1 fields * overlap with the immediate and setting them would overwrite the * immediate we set. */ if (!(brw_eu_inst_src0_type(devinfo, &inst) == BRW_TYPE_DF || brw_eu_inst_src0_type(devinfo, &inst) == BRW_TYPE_UQ || brw_eu_inst_src0_type(devinfo, &inst) == BRW_TYPE_Q)) { brw_eu_inst_set_src1_reg_hw_type(devinfo, &inst, 0); } /* Compacted instructions only have 12-bits (plus 1 for the other 20) * for immediate values. Presumably the hardware engineers realized * that the only useful floating-point value that could be represented * in this format is 0.0, which can also be represented as a VF-typed * immediate, so they gave us the previously mentioned mapping on IVB+. * * Strangely, we do have a mapping for imm:f in src1, so we don't need * to do this there. * * If we see a 0.0:F, change the type to VF so that it can be compacted. * * Compaction of floating-point immediates is improved on Gfx12, thus * removing the need for this. */ if (devinfo->ver < 12 && brw_eu_inst_imm_ud(devinfo, &inst) == 0x0 && brw_eu_inst_src0_type(devinfo, &inst) == BRW_TYPE_F && brw_eu_inst_dst_type(devinfo, &inst) == BRW_TYPE_F && brw_eu_inst_dst_hstride(devinfo, &inst) == BRW_HORIZONTAL_STRIDE_1) { enum brw_reg_file file = brw_eu_inst_src0_reg_file(devinfo, &inst); brw_eu_inst_set_src0_file_type(devinfo, &inst, file, BRW_TYPE_VF); } /* There are no mappings for dst:d | i:d, so if the immediate is suitable * set the types to :UD so the instruction can be compacted. * * FINISHME: Use dst:f | imm:f on Gfx12 */ if (devinfo->ver < 12 && compact_immediate(devinfo, BRW_TYPE_D, brw_eu_inst_imm_ud(devinfo, &inst)) != -1 && brw_eu_inst_cond_modifier(devinfo, &inst) == BRW_CONDITIONAL_NONE && brw_eu_inst_src0_type(devinfo, &inst) == BRW_TYPE_D && brw_eu_inst_dst_type(devinfo, &inst) == BRW_TYPE_D) { enum brw_reg_file src_file = brw_eu_inst_src0_reg_file(devinfo, &inst); enum brw_reg_file dst_file = brw_eu_inst_dst_reg_file(devinfo, &inst); brw_eu_inst_set_src0_file_type(devinfo, &inst, src_file, BRW_TYPE_UD); brw_eu_inst_set_dst_file_type(devinfo, &inst, dst_file, BRW_TYPE_UD); } return inst; } /** * Tries to compact instruction src into dst. * * It doesn't modify dst unless src is compactable, which is relied on by * brw_compact_instructions(). */ static bool try_compact_instruction(const struct compaction_state *c, brw_eu_compact_inst *dst, const brw_eu_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; brw_eu_compact_inst temp; assert(brw_eu_inst_cmpt_control(devinfo, src) == 0); if (is_3src(c->isa, brw_eu_inst_opcode(c->isa, src))) { memset(&temp, 0, sizeof(temp)); if (brw_try_compact_3src_instruction(c->isa, &temp, src)) { *dst = temp; return true; } else { return false; } } enum brw_reg_type type; bool is_immediate = has_immediate(devinfo, src, &type); unsigned compacted_imm = 0; if (is_immediate) { compacted_imm = compact_immediate(devinfo, type, brw_eu_inst_imm_ud(devinfo, src)); if (compacted_imm == -1) return false; } if (has_unmapped_bits(c->isa, src)) return false; memset(&temp, 0, sizeof(temp)); #define compact(field) \ brw_eu_compact_inst_set_##field(devinfo, &temp, brw_eu_inst_##field(devinfo, src)) #define compact_reg(field) \ brw_eu_compact_inst_set_##field##_reg_nr(devinfo, &temp, \ brw_eu_inst_##field##_da_reg_nr(devinfo, src)) compact(hw_opcode); compact(debug_control); if (!set_control_index(c, &temp, src)) return false; if (!set_datatype_index(c, &temp, src, is_immediate)) return false; if (!set_subreg_index(c, &temp, src, is_immediate)) return false; if (!set_src0_index(c, &temp, src)) return false; if (!set_src1_index(c, &temp, src, is_immediate, compacted_imm)) return false; if (devinfo->ver >= 12) { compact(swsb); compact_reg(dst); compact_reg(src0); if (is_immediate) { /* src1 reg takes the high 8 bits (of the 12-bit compacted value) */ brw_eu_compact_inst_set_src1_reg_nr(devinfo, &temp, compacted_imm >> 4); } else { compact_reg(src1); } } else { compact(acc_wr_control); compact(cond_modifier); compact_reg(dst); compact_reg(src0); if (is_immediate) { /* src1 reg takes the low 8 bits (of the 13-bit compacted value) */ brw_eu_compact_inst_set_src1_reg_nr(devinfo, &temp, compacted_imm & 0xff); } else { compact_reg(src1); } } brw_eu_compact_inst_set_cmpt_control(devinfo, &temp, true); #undef compact #undef compact_reg *dst = temp; return true; } bool brw_try_compact_instruction(const struct brw_isa_info *isa, brw_eu_compact_inst *dst, const brw_eu_inst *src) { struct compaction_state c; compaction_state_init(&c, isa); return try_compact_instruction(&c, dst, src); } static void set_uncompacted_control(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; uint32_t uncompacted = c->control_index_table[brw_eu_compact_inst_control_index(devinfo, src)]; if (devinfo->ver >= 20) { brw_eu_inst_set_bits(dst, 95, 92, (uncompacted >> 14) & 0xf); brw_eu_inst_set_bits(dst, 34, 34, (uncompacted >> 13) & 0x1); brw_eu_inst_set_bits(dst, 32, 32, (uncompacted >> 12) & 0x1); brw_eu_inst_set_bits(dst, 31, 31, (uncompacted >> 11) & 0x1); brw_eu_inst_set_bits(dst, 28, 28, (uncompacted >> 10) & 0x1); brw_eu_inst_set_bits(dst, 27, 26, (uncompacted >> 8) & 0x3); brw_eu_inst_set_bits(dst, 25, 24, (uncompacted >> 6) & 0x3); brw_eu_inst_set_bits(dst, 23, 21, (uncompacted >> 3) & 0x7); brw_eu_inst_set_bits(dst, 20, 18, (uncompacted >> 0) & 0x7); } else if (devinfo->ver >= 12) { brw_eu_inst_set_bits(dst, 95, 92, (uncompacted >> 17)); brw_eu_inst_set_bits(dst, 34, 34, (uncompacted >> 16) & 0x1); brw_eu_inst_set_bits(dst, 33, 33, (uncompacted >> 15) & 0x1); brw_eu_inst_set_bits(dst, 32, 32, (uncompacted >> 14) & 0x1); brw_eu_inst_set_bits(dst, 31, 31, (uncompacted >> 13) & 0x1); brw_eu_inst_set_bits(dst, 28, 28, (uncompacted >> 12) & 0x1); brw_eu_inst_set_bits(dst, 27, 24, (uncompacted >> 8) & 0xf); brw_eu_inst_set_bits(dst, 23, 22, (uncompacted >> 6) & 0x3); brw_eu_inst_set_bits(dst, 21, 19, (uncompacted >> 3) & 0x7); brw_eu_inst_set_bits(dst, 18, 16, (uncompacted >> 0) & 0x7); } else { brw_eu_inst_set_bits(dst, 33, 31, (uncompacted >> 16)); brw_eu_inst_set_bits(dst, 23, 12, (uncompacted >> 4) & 0xfff); brw_eu_inst_set_bits(dst, 10, 9, (uncompacted >> 2) & 0x3); brw_eu_inst_set_bits(dst, 34, 34, (uncompacted >> 1) & 0x1); brw_eu_inst_set_bits(dst, 8, 8, (uncompacted >> 0) & 0x1); } } static void set_uncompacted_datatype(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; uint32_t uncompacted = c->datatype_table[brw_eu_compact_inst_datatype_index(devinfo, src)]; if (devinfo->ver >= 12) { brw_eu_inst_set_bits(dst, 98, 98, (uncompacted >> 19)); brw_eu_inst_set_bits(dst, 91, 88, (uncompacted >> 15) & 0xf); brw_eu_inst_set_bits(dst, 66, 66, (uncompacted >> 14) & 0x1); brw_eu_inst_set_bits(dst, 50, 50, (uncompacted >> 13) & 0x1); brw_eu_inst_set_bits(dst, 49, 48, (uncompacted >> 11) & 0x3); brw_eu_inst_set_bits(dst, 47, 47, (uncompacted >> 10) & 0x1); brw_eu_inst_set_bits(dst, 46, 46, (uncompacted >> 9) & 0x1); brw_eu_inst_set_bits(dst, 43, 40, (uncompacted >> 5) & 0xf); brw_eu_inst_set_bits(dst, 39, 36, (uncompacted >> 1) & 0xf); brw_eu_inst_set_bits(dst, 35, 35, (uncompacted >> 0) & 0x1); } else { brw_eu_inst_set_bits(dst, 63, 61, (uncompacted >> 18)); brw_eu_inst_set_bits(dst, 94, 89, (uncompacted >> 12) & 0x3f); brw_eu_inst_set_bits(dst, 46, 35, (uncompacted >> 0) & 0xfff); } } static void set_uncompacted_subreg(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; uint16_t uncompacted = c->subreg_table[brw_eu_compact_inst_subreg_index(devinfo, src)]; if (devinfo->ver >= 20) { brw_eu_inst_set_bits(dst, 33, 33, (uncompacted >> 0) & 0x1); brw_eu_inst_set_bits(dst, 55, 51, (uncompacted >> 1) & 0x1f); brw_eu_inst_set_bits(dst, 71, 67, (uncompacted >> 6) & 0x1f); brw_eu_inst_set_bits(dst, 87, 87, (uncompacted >> 11) & 0x1); } else if (devinfo->ver >= 12) { brw_eu_inst_set_bits(dst, 103, 99, (uncompacted >> 10)); brw_eu_inst_set_bits(dst, 71, 67, (uncompacted >> 5) & 0x1f); brw_eu_inst_set_bits(dst, 55, 51, (uncompacted >> 0) & 0x1f); } else { brw_eu_inst_set_bits(dst, 100, 96, (uncompacted >> 10)); brw_eu_inst_set_bits(dst, 68, 64, (uncompacted >> 5) & 0x1f); brw_eu_inst_set_bits(dst, 52, 48, (uncompacted >> 0) & 0x1f); } } static void set_uncompacted_src0(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; uint32_t compacted = brw_eu_compact_inst_src0_index(devinfo, src); uint16_t uncompacted = c->src0_index_table[compacted]; if (devinfo->ver >= 12) { if (devinfo->ver < 20) brw_eu_inst_set_bits(dst, 87, 87, (uncompacted >> 11) & 0x1); brw_eu_inst_set_bits(dst, 86, 84, (uncompacted >> 8) & 0x7); brw_eu_inst_set_bits(dst, 83, 81, (uncompacted >> 5) & 0x7); brw_eu_inst_set_bits(dst, 80, 80, (uncompacted >> 4) & 0x1); brw_eu_inst_set_bits(dst, 65, 64, (uncompacted >> 2) & 0x3); brw_eu_inst_set_bits(dst, 45, 44, (uncompacted >> 0) & 0x3); } else { brw_eu_inst_set_bits(dst, 88, 77, uncompacted); } } static void set_uncompacted_src1(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; uint16_t uncompacted = c->src1_index_table[brw_eu_compact_inst_src1_index(devinfo, src)]; if (devinfo->ver >= 20) { brw_eu_inst_set_bits(dst, 121, 120, (uncompacted >> 14) & 0x3); brw_eu_inst_set_bits(dst, 118, 116, (uncompacted >> 11) & 0x7); brw_eu_inst_set_bits(dst, 115, 113, (uncompacted >> 8) & 0x7); brw_eu_inst_set_bits(dst, 112, 112, (uncompacted >> 7) & 0x1); brw_eu_inst_set_bits(dst, 103, 99, (uncompacted >> 2) & 0x1f); brw_eu_inst_set_bits(dst, 97, 96, (uncompacted >> 0) & 0x3); } else if (devinfo->ver >= 12) { brw_eu_inst_set_bits(dst, 121, 120, (uncompacted >> 10)); brw_eu_inst_set_bits(dst, 119, 116, (uncompacted >> 6) & 0xf); brw_eu_inst_set_bits(dst, 115, 113, (uncompacted >> 3) & 0x7); brw_eu_inst_set_bits(dst, 112, 112, (uncompacted >> 2) & 0x1); brw_eu_inst_set_bits(dst, 97, 96, (uncompacted >> 0) & 0x3); } else { brw_eu_inst_set_bits(dst, 120, 109, uncompacted); } } static void set_uncompacted_3src_control_index(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src, bool is_dpas) { const struct intel_device_info *devinfo = c->isa->devinfo; if (devinfo->ver >= 20) { uint64_t compacted = brw_eu_compact_inst_3src_control_index(devinfo, src); uint64_t uncompacted = is_dpas ? xe2_3src_dpas_control_index_table[compacted] : xe2_3src_control_index_table[compacted]; brw_eu_inst_set_bits(dst, 95, 92, (uncompacted >> 30) & 0xf); brw_eu_inst_set_bits(dst, 90, 88, (uncompacted >> 27) & 0x7); brw_eu_inst_set_bits(dst, 82, 80, (uncompacted >> 24) & 0x7); brw_eu_inst_set_bits(dst, 50, 50, (uncompacted >> 23) & 0x1); brw_eu_inst_set_bits(dst, 49, 48, (uncompacted >> 21) & 0x3); brw_eu_inst_set_bits(dst, 42, 40, (uncompacted >> 18) & 0x7); brw_eu_inst_set_bits(dst, 39, 39, (uncompacted >> 17) & 0x1); brw_eu_inst_set_bits(dst, 38, 36, (uncompacted >> 14) & 0x7); brw_eu_inst_set_bits(dst, 34, 34, (uncompacted >> 13) & 0x1); brw_eu_inst_set_bits(dst, 32, 32, (uncompacted >> 12) & 0x1); brw_eu_inst_set_bits(dst, 31, 31, (uncompacted >> 11) & 0x1); brw_eu_inst_set_bits(dst, 28, 28, (uncompacted >> 10) & 0x1); brw_eu_inst_set_bits(dst, 27, 26, (uncompacted >> 8) & 0x3); brw_eu_inst_set_bits(dst, 25, 24, (uncompacted >> 6) & 0x3); brw_eu_inst_set_bits(dst, 23, 21, (uncompacted >> 3) & 0x7); brw_eu_inst_set_bits(dst, 20, 18, (uncompacted >> 0) & 0x7); } else if (devinfo->verx10 >= 125) { uint64_t compacted = brw_eu_compact_inst_3src_control_index(devinfo, src); uint64_t uncompacted = xehp_3src_control_index_table[compacted]; brw_eu_inst_set_bits(dst, 95, 92, (uncompacted >> 33)); brw_eu_inst_set_bits(dst, 90, 88, (uncompacted >> 30) & 0x7); brw_eu_inst_set_bits(dst, 82, 80, (uncompacted >> 27) & 0x7); brw_eu_inst_set_bits(dst, 50, 50, (uncompacted >> 26) & 0x1); brw_eu_inst_set_bits(dst, 49, 48, (uncompacted >> 24) & 0x3); brw_eu_inst_set_bits(dst, 42, 40, (uncompacted >> 21) & 0x7); brw_eu_inst_set_bits(dst, 39, 39, (uncompacted >> 20) & 0x1); brw_eu_inst_set_bits(dst, 38, 36, (uncompacted >> 17) & 0x7); brw_eu_inst_set_bits(dst, 34, 34, (uncompacted >> 16) & 0x1); brw_eu_inst_set_bits(dst, 33, 33, (uncompacted >> 15) & 0x1); brw_eu_inst_set_bits(dst, 32, 32, (uncompacted >> 14) & 0x1); brw_eu_inst_set_bits(dst, 31, 31, (uncompacted >> 13) & 0x1); brw_eu_inst_set_bits(dst, 28, 28, (uncompacted >> 12) & 0x1); brw_eu_inst_set_bits(dst, 27, 24, (uncompacted >> 8) & 0xf); brw_eu_inst_set_bits(dst, 23, 23, (uncompacted >> 7) & 0x1); brw_eu_inst_set_bits(dst, 22, 22, (uncompacted >> 6) & 0x1); brw_eu_inst_set_bits(dst, 21, 19, (uncompacted >> 3) & 0x7); brw_eu_inst_set_bits(dst, 18, 16, (uncompacted >> 0) & 0x7); } else if (devinfo->ver >= 12) { uint64_t compacted = brw_eu_compact_inst_3src_control_index(devinfo, src); uint64_t uncompacted = gfx12_3src_control_index_table[compacted]; brw_eu_inst_set_bits(dst, 95, 92, (uncompacted >> 32)); brw_eu_inst_set_bits(dst, 90, 88, (uncompacted >> 29) & 0x7); brw_eu_inst_set_bits(dst, 82, 80, (uncompacted >> 26) & 0x7); brw_eu_inst_set_bits(dst, 50, 50, (uncompacted >> 25) & 0x1); brw_eu_inst_set_bits(dst, 48, 48, (uncompacted >> 24) & 0x1); brw_eu_inst_set_bits(dst, 42, 40, (uncompacted >> 21) & 0x7); brw_eu_inst_set_bits(dst, 39, 39, (uncompacted >> 20) & 0x1); brw_eu_inst_set_bits(dst, 38, 36, (uncompacted >> 17) & 0x7); brw_eu_inst_set_bits(dst, 34, 34, (uncompacted >> 16) & 0x1); brw_eu_inst_set_bits(dst, 33, 33, (uncompacted >> 15) & 0x1); brw_eu_inst_set_bits(dst, 32, 32, (uncompacted >> 14) & 0x1); brw_eu_inst_set_bits(dst, 31, 31, (uncompacted >> 13) & 0x1); brw_eu_inst_set_bits(dst, 28, 28, (uncompacted >> 12) & 0x1); brw_eu_inst_set_bits(dst, 27, 24, (uncompacted >> 8) & 0xf); brw_eu_inst_set_bits(dst, 23, 23, (uncompacted >> 7) & 0x1); brw_eu_inst_set_bits(dst, 22, 22, (uncompacted >> 6) & 0x1); brw_eu_inst_set_bits(dst, 21, 19, (uncompacted >> 3) & 0x7); brw_eu_inst_set_bits(dst, 18, 16, (uncompacted >> 0) & 0x7); } else { uint32_t compacted = brw_eu_compact_inst_3src_control_index(devinfo, src); uint32_t uncompacted = gfx8_3src_control_index_table[compacted]; brw_eu_inst_set_bits(dst, 34, 32, (uncompacted >> 21) & 0x7); brw_eu_inst_set_bits(dst, 28, 8, (uncompacted >> 0) & 0x1fffff); brw_eu_inst_set_bits(dst, 36, 35, (uncompacted >> 24) & 0x3); } } static void set_uncompacted_3src_source_index(const struct intel_device_info *devinfo, brw_eu_inst *dst, brw_eu_compact_inst *src, bool is_dpas) { uint32_t compacted = brw_eu_compact_inst_3src_source_index(devinfo, src); if (devinfo->ver >= 12) { const uint32_t *three_src_source_index_table = devinfo->ver >= 20 ? (is_dpas ? xe2_3src_dpas_source_index_table : xe2_3src_source_index_table) : devinfo->verx10 >= 125 ? xehp_3src_source_index_table : gfx12_3src_source_index_table; uint32_t uncompacted = three_src_source_index_table[compacted]; brw_eu_inst_set_bits(dst, 114, 114, (uncompacted >> 20)); brw_eu_inst_set_bits(dst, 113, 112, (uncompacted >> 18) & 0x3); brw_eu_inst_set_bits(dst, 98, 98, (uncompacted >> 17) & 0x1); brw_eu_inst_set_bits(dst, 97, 96, (uncompacted >> 15) & 0x3); brw_eu_inst_set_bits(dst, 91, 91, (uncompacted >> 14) & 0x1); brw_eu_inst_set_bits(dst, 87, 86, (uncompacted >> 12) & 0x3); brw_eu_inst_set_bits(dst, 85, 84, (uncompacted >> 10) & 0x3); brw_eu_inst_set_bits(dst, 83, 83, (uncompacted >> 9) & 0x1); brw_eu_inst_set_bits(dst, 66, 66, (uncompacted >> 8) & 0x1); brw_eu_inst_set_bits(dst, 65, 64, (uncompacted >> 6) & 0x3); brw_eu_inst_set_bits(dst, 47, 47, (uncompacted >> 5) & 0x1); brw_eu_inst_set_bits(dst, 46, 46, (uncompacted >> 4) & 0x1); brw_eu_inst_set_bits(dst, 45, 44, (uncompacted >> 2) & 0x3); brw_eu_inst_set_bits(dst, 43, 43, (uncompacted >> 1) & 0x1); brw_eu_inst_set_bits(dst, 35, 35, (uncompacted >> 0) & 0x1); } else { uint64_t uncompacted = gfx8_3src_source_index_table[compacted]; brw_eu_inst_set_bits(dst, 83, 83, (uncompacted >> 43) & 0x1); brw_eu_inst_set_bits(dst, 114, 107, (uncompacted >> 35) & 0xff); brw_eu_inst_set_bits(dst, 93, 86, (uncompacted >> 27) & 0xff); brw_eu_inst_set_bits(dst, 72, 65, (uncompacted >> 19) & 0xff); brw_eu_inst_set_bits(dst, 55, 37, (uncompacted >> 0) & 0x7ffff); brw_eu_inst_set_bits(dst, 126, 125, (uncompacted >> 47) & 0x3); brw_eu_inst_set_bits(dst, 105, 104, (uncompacted >> 45) & 0x3); brw_eu_inst_set_bits(dst, 84, 84, (uncompacted >> 44) & 0x1); } } static void set_uncompacted_3src_subreg_index(const struct intel_device_info *devinfo, brw_eu_inst *dst, brw_eu_compact_inst *src) { assert(devinfo->ver >= 12); uint32_t compacted = brw_eu_compact_inst_3src_subreg_index(devinfo, src); uint32_t uncompacted = (devinfo->ver >= 20 ? xe2_3src_subreg_table[compacted]: gfx12_3src_subreg_table[compacted]); brw_eu_inst_set_bits(dst, 119, 115, (uncompacted >> 15)); brw_eu_inst_set_bits(dst, 103, 99, (uncompacted >> 10) & 0x1f); brw_eu_inst_set_bits(dst, 71, 67, (uncompacted >> 5) & 0x1f); brw_eu_inst_set_bits(dst, 55, 51, (uncompacted >> 0) & 0x1f); } static void brw_uncompact_3src_instruction(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src, bool is_dpas) { const struct intel_device_info *devinfo = c->isa->devinfo; #define uncompact(field) \ brw_eu_inst_set_3src_##field(devinfo, dst, brw_eu_compact_inst_3src_##field(devinfo, src)) #define uncompact_a16(field) \ brw_eu_inst_set_3src_a16_##field(devinfo, dst, brw_eu_compact_inst_3src_##field(devinfo, src)) uncompact(hw_opcode); if (devinfo->ver >= 12) { set_uncompacted_3src_control_index(c, dst, src, is_dpas); set_uncompacted_3src_source_index(devinfo, dst, src, is_dpas); set_uncompacted_3src_subreg_index(devinfo, dst, src); uncompact(debug_control); uncompact(swsb); uncompact(dst_reg_nr); uncompact(src0_reg_nr); uncompact(src1_reg_nr); uncompact(src2_reg_nr); } else { set_uncompacted_3src_control_index(c, dst, src, is_dpas); set_uncompacted_3src_source_index(devinfo, dst, src, is_dpas); uncompact(dst_reg_nr); uncompact_a16(src0_rep_ctrl); uncompact(debug_control); uncompact(saturate); uncompact_a16(src1_rep_ctrl); uncompact_a16(src2_rep_ctrl); uncompact(src0_reg_nr); uncompact(src1_reg_nr); uncompact(src2_reg_nr); uncompact_a16(src0_subreg_nr); uncompact_a16(src1_subreg_nr); uncompact_a16(src2_subreg_nr); } brw_eu_inst_set_3src_cmpt_control(devinfo, dst, false); #undef uncompact #undef uncompact_a16 } static void uncompact_instruction(const struct compaction_state *c, brw_eu_inst *dst, brw_eu_compact_inst *src) { const struct intel_device_info *devinfo = c->isa->devinfo; memset(dst, 0, sizeof(*dst)); const enum opcode opcode = brw_opcode_decode(c->isa, brw_eu_compact_inst_3src_hw_opcode(devinfo, src)); if (is_3src(c->isa, opcode)) { const bool is_dpas = opcode == BRW_OPCODE_DPAS; brw_uncompact_3src_instruction(c, dst, src, is_dpas); return; } #define uncompact(field) \ brw_eu_inst_set_##field(devinfo, dst, brw_eu_compact_inst_##field(devinfo, src)) #define uncompact_reg(field) \ brw_eu_inst_set_##field##_da_reg_nr(devinfo, dst, \ brw_eu_compact_inst_##field##_reg_nr(devinfo, src)) uncompact(hw_opcode); uncompact(debug_control); set_uncompacted_control(c, dst, src); set_uncompacted_datatype(c, dst, src); set_uncompacted_subreg(c, dst, src); set_uncompacted_src0(c, dst, src); enum brw_reg_type type; if (has_immediate(devinfo, dst, &type)) { unsigned imm = uncompact_immediate(devinfo, type, brw_eu_compact_inst_imm(devinfo, src)); brw_eu_inst_set_imm_ud(devinfo, dst, imm); } else { set_uncompacted_src1(c, dst, src); uncompact_reg(src1); } if (devinfo->ver >= 12) { uncompact(swsb); uncompact_reg(dst); uncompact_reg(src0); } else { uncompact(acc_wr_control); uncompact(cond_modifier); uncompact_reg(dst); uncompact_reg(src0); } brw_eu_inst_set_cmpt_control(devinfo, dst, false); #undef uncompact #undef uncompact_reg } void brw_uncompact_instruction(const struct brw_isa_info *isa, brw_eu_inst *dst, brw_eu_compact_inst *src) { struct compaction_state c; compaction_state_init(&c, isa); uncompact_instruction(&c, dst, src); } void brw_debug_compact_uncompact(const struct brw_isa_info *isa, brw_eu_inst *orig, brw_eu_inst *uncompacted) { fprintf(stderr, "Instruction compact/uncompact changed (gen%d):\n", isa->devinfo->ver); fprintf(stderr, " before: "); brw_disassemble_inst(stderr, isa, orig, true, 0, NULL); fprintf(stderr, " after: "); brw_disassemble_inst(stderr, isa, uncompacted, false, 0, NULL); uint32_t *before_bits = (uint32_t *)orig; uint32_t *after_bits = (uint32_t *)uncompacted; fprintf(stderr, " changed bits:\n"); for (int i = 0; i < 128; i++) { uint32_t before = before_bits[i / 32] & (1 << (i & 31)); uint32_t after = after_bits[i / 32] & (1 << (i & 31)); if (before != after) { fprintf(stderr, " bit %d, %s to %s\n", i, before ? "set" : "unset", after ? "set" : "unset"); } } } static int compacted_between(int old_ip, int old_target_ip, int *compacted_counts) { int this_compacted_count = compacted_counts[old_ip]; int target_compacted_count = compacted_counts[old_target_ip]; return target_compacted_count - this_compacted_count; } static void update_uip_jip(const struct brw_isa_info *isa, brw_eu_inst *insn, int this_old_ip, int *compacted_counts) { const struct intel_device_info *devinfo = isa->devinfo; /* JIP and UIP are in units of bytes on Gfx8+. */ int shift = 3; /* Even though the values are signed, we don't need the rounding behavior * of integer division. The shifts are safe. */ assert(brw_eu_inst_jip(devinfo, insn) % 8 == 0 && brw_eu_inst_uip(devinfo, insn) % 8 == 0); int32_t jip_compacted = brw_eu_inst_jip(devinfo, insn) >> shift; jip_compacted -= compacted_between(this_old_ip, this_old_ip + (jip_compacted / 2), compacted_counts); brw_eu_inst_set_jip(devinfo, insn, (uint32_t)jip_compacted << shift); if (brw_eu_inst_opcode(isa, insn) == BRW_OPCODE_ENDIF || brw_eu_inst_opcode(isa, insn) == BRW_OPCODE_WHILE) return; int32_t uip_compacted = brw_eu_inst_uip(devinfo, insn) >> shift; uip_compacted -= compacted_between(this_old_ip, this_old_ip + (uip_compacted / 2), compacted_counts); brw_eu_inst_set_uip(devinfo, insn, (uint32_t)uip_compacted << shift); } static void compaction_state_init(struct compaction_state *c, const struct brw_isa_info *isa) { const struct intel_device_info *devinfo = isa->devinfo; assert(g45_subreg_table[ARRAY_SIZE(g45_subreg_table) - 1] != 0); assert(gfx8_control_index_table[ARRAY_SIZE(gfx8_control_index_table) - 1] != 0); assert(gfx8_datatype_table[ARRAY_SIZE(gfx8_datatype_table) - 1] != 0); assert(gfx8_subreg_table[ARRAY_SIZE(gfx8_subreg_table) - 1] != 0); assert(gfx8_src_index_table[ARRAY_SIZE(gfx8_src_index_table) - 1] != 0); assert(gfx11_datatype_table[ARRAY_SIZE(gfx11_datatype_table) - 1] != 0); assert(gfx12_control_index_table[ARRAY_SIZE(gfx12_control_index_table) - 1] != 0); assert(gfx12_datatype_table[ARRAY_SIZE(gfx12_datatype_table) - 1] != 0); assert(gfx12_subreg_table[ARRAY_SIZE(gfx12_subreg_table) - 1] != 0); assert(gfx12_src0_index_table[ARRAY_SIZE(gfx12_src0_index_table) - 1] != 0); assert(gfx12_src1_index_table[ARRAY_SIZE(gfx12_src1_index_table) - 1] != 0); assert(xehp_src0_index_table[ARRAY_SIZE(xehp_src0_index_table) - 1] != 0); assert(xehp_src1_index_table[ARRAY_SIZE(xehp_src1_index_table) - 1] != 0); assert(xe2_control_index_table[ARRAY_SIZE(xe2_control_index_table) - 1] != 0); assert(xe2_datatype_table[ARRAY_SIZE(xe2_datatype_table) - 1] != 0); assert(xe2_subreg_table[ARRAY_SIZE(xe2_subreg_table) - 1] != 0); assert(xe2_src0_index_table[ARRAY_SIZE(xe2_src0_index_table) - 1] != 0); assert(xe2_src1_index_table[ARRAY_SIZE(xe2_src1_index_table) - 1] != 0); c->isa = isa; switch (devinfo->ver) { case 20: case 30: c->control_index_table = xe2_control_index_table; c->datatype_table = xe2_datatype_table; c->subreg_table = xe2_subreg_table; c->src0_index_table = xe2_src0_index_table; c->src1_index_table = xe2_src1_index_table; break; case 12: c->control_index_table = gfx12_control_index_table;; c->datatype_table = gfx12_datatype_table; c->subreg_table = gfx12_subreg_table; if (devinfo->verx10 >= 125) { c->src0_index_table = xehp_src0_index_table; c->src1_index_table = xehp_src1_index_table; } else { c->src0_index_table = gfx12_src0_index_table; c->src1_index_table = gfx12_src1_index_table; } break; case 11: c->control_index_table = gfx8_control_index_table; c->datatype_table = gfx11_datatype_table; c->subreg_table = gfx8_subreg_table; c->src0_index_table = gfx8_src_index_table; c->src1_index_table = gfx8_src_index_table; break; case 9: c->control_index_table = gfx8_control_index_table; c->datatype_table = gfx8_datatype_table; c->subreg_table = gfx8_subreg_table; c->src0_index_table = gfx8_src_index_table; c->src1_index_table = gfx8_src_index_table; break; default: unreachable("unknown generation"); } } void brw_compact_instructions(struct brw_codegen *p, int start_offset, struct disasm_info *disasm) { if (INTEL_DEBUG(DEBUG_NO_COMPACTION)) return; const struct intel_device_info *devinfo = p->devinfo; void *store = p->store + start_offset / 16; /* For an instruction at byte offset 16*i before compaction, this is the * number of compacted instructions minus the number of padding NOP/NENOPs * that preceded it. */ unsigned num_compacted_counts = (p->next_insn_offset - start_offset) / sizeof(brw_eu_inst); int *compacted_counts = calloc(num_compacted_counts, sizeof(*compacted_counts)); /* For an instruction at byte offset 8*i after compaction, this was its IP * (in 16-byte units) before compaction. */ unsigned num_old_ip = (p->next_insn_offset - start_offset) / sizeof(brw_eu_compact_inst) + 1; int *old_ip = calloc(num_old_ip, sizeof(*old_ip)); struct compaction_state c; compaction_state_init(&c, p->isa); int offset = 0; int compacted_count = 0; for (int src_offset = 0; src_offset < p->next_insn_offset - start_offset; src_offset += sizeof(brw_eu_inst)) { brw_eu_inst *src = store + src_offset; void *dst = store + offset; old_ip[offset / sizeof(brw_eu_compact_inst)] = src_offset / sizeof(brw_eu_inst); compacted_counts[src_offset / sizeof(brw_eu_inst)] = compacted_count; brw_eu_inst inst = precompact(p->isa, *src); brw_eu_inst saved = inst; if (try_compact_instruction(&c, dst, &inst)) { compacted_count++; if (INTEL_DEBUG(DEBUG_VS | DEBUG_GS | DEBUG_TCS | DEBUG_TASK | DEBUG_WM | DEBUG_CS | DEBUG_TES | DEBUG_MESH | DEBUG_RT)) { brw_eu_inst uncompacted; uncompact_instruction(&c, &uncompacted, dst); if (memcmp(&saved, &uncompacted, sizeof(uncompacted))) { brw_debug_compact_uncompact(p->isa, &saved, &uncompacted); } } offset += sizeof(brw_eu_compact_inst); } else { /* If we didn't compact this instruction, we need to move it down into * place. */ if (offset != src_offset) { memmove(dst, src, sizeof(brw_eu_inst)); } offset += sizeof(brw_eu_inst); } } /* Add an entry for the ending offset of the program. This greatly * simplifies the linked list walk at the end of the function. */ old_ip[offset / sizeof(brw_eu_compact_inst)] = (p->next_insn_offset - start_offset) / sizeof(brw_eu_inst); /* Fix up control flow offsets. */ p->next_insn_offset = start_offset + offset; for (offset = 0; offset < p->next_insn_offset - start_offset; offset = next_offset(p, store, offset)) { brw_eu_inst *insn = store + offset; int this_old_ip = old_ip[offset / sizeof(brw_eu_compact_inst)]; int this_compacted_count = compacted_counts[this_old_ip]; switch (brw_eu_inst_opcode(p->isa, insn)) { case BRW_OPCODE_BREAK: case BRW_OPCODE_CONTINUE: case BRW_OPCODE_HALT: update_uip_jip(p->isa, insn, this_old_ip, compacted_counts); break; case BRW_OPCODE_IF: case BRW_OPCODE_ELSE: case BRW_OPCODE_ENDIF: case BRW_OPCODE_WHILE: if (brw_eu_inst_cmpt_control(devinfo, insn)) { brw_eu_inst uncompacted; uncompact_instruction(&c, &uncompacted, (brw_eu_compact_inst *)insn); update_uip_jip(p->isa, &uncompacted, this_old_ip, compacted_counts); bool ret = try_compact_instruction(&c, (brw_eu_compact_inst *)insn, &uncompacted); assert(ret); (void)ret; } else { update_uip_jip(p->isa, insn, this_old_ip, compacted_counts); } break; case BRW_OPCODE_ADD: /* Add instructions modifying the IP register use an immediate src1, * and Gens that use this cannot compact instructions with immediate * operands. */ if (brw_eu_inst_cmpt_control(devinfo, insn)) break; if (brw_eu_inst_dst_reg_file(devinfo, insn) == ARF && brw_eu_inst_dst_da_reg_nr(devinfo, insn) == BRW_ARF_IP) { assert(brw_eu_inst_src1_reg_file(devinfo, insn) == IMM); int shift = 3; int jump_compacted = brw_eu_inst_imm_d(devinfo, insn) >> shift; int target_old_ip = this_old_ip + (jump_compacted / 2); int target_compacted_count = compacted_counts[target_old_ip]; jump_compacted -= (target_compacted_count - this_compacted_count); brw_eu_inst_set_imm_ud(devinfo, insn, jump_compacted << shift); } break; default: break; } } /* p->nr_insn is counting the number of uncompacted instructions still, so * divide. We do want to be sure there's a valid instruction in any * alignment padding, so that the next compression pass (for the FS 8/16 * compile passes) parses correctly. */ if (p->next_insn_offset & sizeof(brw_eu_compact_inst)) { brw_eu_compact_inst *align = store + offset; memset(align, 0, sizeof(*align)); brw_eu_compact_inst_set_hw_opcode( devinfo, align, brw_opcode_encode(p->isa, BRW_OPCODE_NOP)); brw_eu_compact_inst_set_cmpt_control(devinfo, align, true); p->next_insn_offset += sizeof(brw_eu_compact_inst); } p->nr_insn = p->next_insn_offset / sizeof(brw_eu_inst); for (int i = 0; i < p->num_relocs; i++) { if (p->relocs[i].offset < (uint32_t)start_offset) continue; assert(p->relocs[i].offset % 16 == 0); unsigned idx = (p->relocs[i].offset - start_offset) / 16; p->relocs[i].offset -= compacted_counts[idx] * 8; } /* Update the instruction offsets for each group. */ if (disasm) { int offset = 0; foreach_list_typed(struct inst_group, group, link, &disasm->group_list) { while (start_offset + old_ip[offset / sizeof(brw_eu_compact_inst)] * sizeof(brw_eu_inst) != group->offset) { assert(start_offset + old_ip[offset / sizeof(brw_eu_compact_inst)] * sizeof(brw_eu_inst) < group->offset); offset = next_offset(p, store, offset); } group->offset = start_offset + offset; } } free(compacted_counts); free(old_ip); }