gen_builtin_symbols.py

Branch :
Show log
Commit
Author : Olli Etuaho
Date : 2018-08-23 14:10:02
Hash : 8ca60805
Message : Add 2D MS array sampler support to compiler This also places textureSize(gsampler2DMS) correctly in the ESSL 3.10 builtins instead of ESSL 3.00 builtins. BUG=angleproject:2775 TEST=angle_unittests Change-Id: Ieb0f7a7424a5558a5569af6d4fcbcc9b12ec9840 Reviewed-on: https://chromium-review.googlesource.com/1186466 Commit-Queue: Olli Etuaho <oetuaho@nvidia.com> Reviewed-by: Jamie Madill <jmadill@chromium.org>
src/compiler/translator/gen_builtin_symbols.py
#!/usr/bin/python
# Copyright 2018 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# gen_builtin_symbols.py:
#  Code generation for the built-in symbol tables.

from collections import OrderedDict
from datetime import date
import argparse
import hashlib
import json
import re
import os
import sys

def set_working_dir():
    script_dir = os.path.dirname(os.path.abspath(__file__))
    os.chdir(script_dir)

set_working_dir()

variables_json_filename = 'builtin_variables.json'
functions_txt_filename = 'builtin_function_declarations.txt'
hash_filename = 'builtin_symbols_hash_autogen.txt'

all_inputs = [os.path.abspath(__file__), variables_json_filename, functions_txt_filename]
# This script takes a while to run since it searches for hash collisions of mangled names. To avoid
# running it unnecessarily, we first check if we've already ran it with the same inputs.
m = hashlib.md5()
for input_path in all_inputs:
    with open(input_path, 'rU') as input_file:
        m.update(input_file.read())
input_hash = m.hexdigest()
if os.path.exists(hash_filename):
    with open(hash_filename) as hash_file:
        if input_hash == hash_file.read():
            print "Canceling ESSL static builtins code generator - generated hash matches inputs."
            sys.exit(0)

parser = argparse.ArgumentParser()
parser.add_argument('--dump-intermediate-json', help='Dump parsed function data as a JSON file builtin_functions.json', action="store_true")
args = parser.parse_args()

template_immutablestringtest_cpp = """// GENERATED FILE - DO NOT EDIT.
// Generated by {script_name} using data from {function_data_source_name}.
//
// Copyright {copyright_year} The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// ImmutableString_test_autogen.cpp:
//   Tests for matching script-generated hashes with runtime computed hashes.

#include "compiler/translator/ImmutableString.h"
#include "gtest/gtest.h"

namespace sh
{{

TEST(ImmutableStringTest, ScriptGeneratedHashesMatch)
{{
{script_generated_hash_tests}
}}

}}  // namespace sh
"""

# The header file has a "get" function for each variable. They are used in traversers.
# It also declares id values of built-ins with human readable names, so they can be used to identify built-ins.
template_builtin_header = """// GENERATED FILE - DO NOT EDIT.
// Generated by {script_name} using data from {variable_data_source_name} and
// {function_data_source_name}.
//
// Copyright {copyright_year} The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// BuiltIn_autogen.h:
//   Compile-time initialized built-ins.

#ifndef COMPILER_TRANSLATOR_TREEUTIL_BUILTIN_AUTOGEN_H_
#define COMPILER_TRANSLATOR_TREEUTIL_BUILTIN_AUTOGEN_H_

#include "compiler/translator/SymbolUniqueId.h"

namespace sh
{{

class TVariable;

class BuiltInId
{{
public:

{builtin_id_declarations}

}};  // class BuiltInId

namespace BuiltInVariable
{{

{get_variable_declarations}

}}  // namespace BuiltInVariable

}}  // namespace sh

#endif  // COMPILER_TRANSLATOR_TREEUTIL_BUILTIN_AUTOGEN_H_
"""

template_symboltable_h = """// GENERATED FILE - DO NOT EDIT.
// Generated by {script_name} using data from {variable_data_source_name} and
// {function_data_source_name}.
//
// Copyright {copyright_year} The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// SymbolTable_autogen.h:
//   Autogenerated member variables of TSymbolTable.

#ifndef COMPILER_TRANSLATOR_SYMBOLTABLE_AUTOGEN_H_
#define COMPILER_TRANSLATOR_SYMBOLTABLE_AUTOGEN_H_

namespace sh
{{

class TSymbolTableBase
{{
  protected:
    TSymbolTableBase() = default;
{declare_member_variables}
}};

}}  // namespace sh

#endif  // COMPILER_TRANSLATOR_SYMBOLTABLE_AUTOGEN_H_
"""

# By having the variables defined in a cpp file we ensure that there's just one instance of each of the declared variables.
template_symboltable_cpp = """// GENERATED FILE - DO NOT EDIT.
// Generated by {script_name} using data from {variable_data_source_name} and
// {function_data_source_name}.
//
// Copyright {copyright_year} The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// SymbolTable_autogen.cpp:
//   Compile-time initialized built-ins.

#include "compiler/translator/SymbolTable.h"

#include "angle_gl.h"
#include "compiler/translator/tree_util/BuiltIn_autogen.h"
#include "compiler/translator/ImmutableString.h"
#include "compiler/translator/StaticType.h"
#include "compiler/translator/Symbol.h"
#include "compiler/translator/SymbolTable.h"

namespace sh
{{

// Since some of the BuiltInId declarations are used outside of constexpr expressions, we need to
// have these definitions without an initializer. C++17 should eventually remove the need for this.
{builtin_id_definitions}

const int TSymbolTable::kLastBuiltInId = {last_builtin_id};

namespace BuiltInName
{{

constexpr const ImmutableString _empty("");
{name_declarations}

}}  // namespace BuiltInName

// TODO(oetuaho): Would be nice to make this a class instead of a namespace so that we could friend
// this from TVariable. Now symbol constructors taking an id have to be public even though they're
// not supposed to be accessible from outside of here. http://anglebug.com/2390
namespace BuiltInVariable
{{

{variable_declarations}

{get_variable_definitions}

}};  // namespace BuiltInVariable

namespace BuiltInParameters
{{

{parameter_declarations}

}}  // namespace BuiltInParameters

namespace UnmangledBuiltIns
{{

{unmangled_builtin_declarations}

}}  // namespace UnmangledBuiltIns

// TODO(oetuaho): Would be nice to make this a class instead of a namespace so that we could friend
// this from TFunction. Now symbol constructors taking an id have to be public even though they're
// not supposed to be accessible from outside of here. http://anglebug.com/2390
namespace BuiltInFunction
{{

{function_declarations}

}}  // namespace BuiltInFunction

void TSymbolTable::initializeBuiltInVariables(sh::GLenum shaderType,
                                              ShShaderSpec spec,
                                              const ShBuiltInResources &resources)
{{
    const TSourceLoc zeroSourceLoc = {{0, 0, 0, 0}};
{init_member_variables}
}}

const TSymbol *TSymbolTable::findBuiltIn(const ImmutableString &name,
                                         int shaderVersion) const
{{
    if (name.length() > {max_mangled_name_length})
    {{
        return nullptr;
    }}
    uint32_t nameHash = name.mangledNameHash();
    if ((nameHash >> 31) != 0)
    {{
        // The name contains [ or {{.
        return nullptr;
    }}
{get_builtin}
}}

const UnmangledBuiltIn *TSymbolTable::getUnmangledBuiltInForShaderVersion(const ImmutableString &name, int shaderVersion)
{{
    if (name.length() > {max_unmangled_name_length})
    {{
        return nullptr;
    }}
    uint32_t nameHash = name.mangledNameHash();
{get_unmangled_builtin}
}}

}}  // namespace sh
"""

template_parsecontext_header = """// GENERATED FILE - DO NOT EDIT.
// Generated by {script_name} using data from {variable_data_source_name} and
// {function_data_source_name}.
//
// Copyright {copyright_year} The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// ParseContext_autogen.h:
//   Helpers for built-in related checks.

#ifndef COMPILER_TRANSLATOR_PARSECONTEXT_AUTOGEN_H_
#define COMPILER_TRANSLATOR_PARSECONTEXT_AUTOGEN_H_

namespace sh
{{

namespace BuiltInGroup
{{

{is_in_group_definitions}

}}  // namespace BuiltInGroup

}}  // namespace sh

#endif  // COMPILER_TRANSLATOR_PARSECONTEXT_AUTOGEN_H_

"""

parsed_variables = None

basic_types_enumeration = [
    'Void',
    'Float',
    'Int',
    'UInt',
    'Bool',
    'AtomicCounter',
    'YuvCscStandardEXT',
    'Sampler2D',
    'Sampler3D',
    'SamplerCube',
    'Sampler2DArray',
    'SamplerExternalOES',
    'SamplerExternal2DY2YEXT',
    'Sampler2DRect',
    'Sampler2DMS',
    'Sampler2DMSArray',
    'ISampler2D',
    'ISampler3D',
    'ISamplerCube',
    'ISampler2DArray',
    'ISampler2DMS',
    'ISampler2DMSArray',
    'USampler2D',
    'USampler3D',
    'USamplerCube',
    'USampler2DArray',
    'USampler2DMS',
    'USampler2DMSArray',
    'Sampler2DShadow',
    'SamplerCubeShadow',
    'Sampler2DArrayShadow',
    'Image2D',
    'IImage2D',
    'UImage2D',
    'Image3D',
    'IImage3D',
    'UImage3D',
    'Image2DArray',
    'IImage2DArray',
    'UImage2DArray',
    'ImageCube',
    'IImageCube',
    'UImageCube'
]

def get_basic_mangled_name(basic):
    index = basic_types_enumeration.index(basic)
    if index < 26:
        return chr(ord('A') + index)
    return chr(ord('a') + index - 26)

levels = ['ESSL3_1_BUILTINS', 'ESSL3_BUILTINS', 'ESSL1_BUILTINS', 'COMMON_BUILTINS']

def get_shader_version_condition_for_level(level):
    if level == 'ESSL3_1_BUILTINS':
        return 'shaderVersion >= 310'
    elif level == 'ESSL3_BUILTINS':
        return 'shaderVersion >= 300'
    elif level == 'ESSL1_BUILTINS':
        return 'shaderVersion == 100'
    elif level == 'COMMON_BUILTINS':
        return ''
    else:
        raise Exception('Unsupported symbol table level')

class GroupedList:
    """"Class for storing a list of objects grouped by symbol table level and condition."""
    def __init__(self):
        self.objs = OrderedDict()
        self.max_name_length = 0
        # We need to add all the levels here instead of lazily since they must be in a specific order.
        for l in levels:
            self.objs[l] = OrderedDict()

    def add_obj(self, level, condition, name, obj):
        if (level not in levels):
            raise Exception('Unexpected level: ' + str(level))
        if condition not in self.objs[level]:
            self.objs[level][condition] = OrderedDict()
        self.objs[level][condition][name] = obj
        if len(name) > self.max_name_length:
            self.max_name_length = len(name)

    def has_key(self, level, condition, name):
        if (level not in levels):
            raise Exception('Unexpected level: ' + str(level))
        if condition not in self.objs[level]:
            return False
        return (name in self.objs[level][condition])

    def get(self, level, condition, name):
        if self.has_key(level, condition, name):
            return self.objs[level][condition][name]
        return None

    def get_max_name_length(self):
        return self.max_name_length

    def get_switch_code(self):
        code = []
        for level in levels:
            if len(self.objs[level]) == 0:
                continue
            level_condition = get_shader_version_condition_for_level(level)
            if level_condition != '':
                code.append('if ({condition})\n {{'.format(condition = level_condition))

            for condition, objs in self.objs[level].iteritems():
                if len(objs) > 0:
                    if condition != 'NO_CONDITION':
                        condition_header = '  if ({condition})\n {{'.format(condition = condition)
                        code.append(condition_header.replace('shaderType', 'mShaderType'))

                    switch = {}
                    for name, obj in objs.iteritems():
                        name_hash = mangledNameHash(name)
                        if name_hash not in switch:
                            switch[name_hash] = []
                        switch[name_hash].append(obj['hash_matched_code'])

                    code.append('switch(nameHash) {')
                    for name_hash, obj in sorted(switch.iteritems()):
                        code.append('case 0x' + ('%08x' % name_hash) + 'u:\n{')
                        code += obj
                        code.append('break;\n}')
                    code.append('}')

                    if condition != 'NO_CONDITION':
                        code.append('}')

            if level_condition != '':
                code.append('}')
        code.append('return nullptr;')
        return '\n'.join(code)

class TType:
    def __init__(self, glsl_header_type):
        if isinstance(glsl_header_type, basestring):
            self.data = self.parse_type(glsl_header_type)
        else:
            self.data = glsl_header_type
        self.normalize()

    def normalize(self):
        # Note that this will set primarySize and secondarySize also on genTypes. In that case they
        # are overridden when the specific types are generated.
        if 'primarySize' not in self.data:
            if ('secondarySize' in self.data):
                raise Exception('Unexpected secondarySize on type that does not have primarySize set')
            self.data['primarySize'] = 1
        if 'secondarySize' not in self.data:
            self.data['secondarySize'] = 1
        if 'precision' not in self.data:
            self.data['precision'] = 'Undefined'
        if 'qualifier' not in self.data:
            self.data['qualifier'] = 'Global'

    def get_statictype_string(self):
        template_type = 'StaticType::Get<Ebt{basic}, Ebp{precision}, Evq{qualifier}, {primarySize}, {secondarySize}>()'
        return template_type.format(**self.data)

    def get_dynamic_type_string(self):
        template_type = 'new TType(Ebt{basic}, Ebp{precision}, Evq{qualifier}, {primarySize}, {secondarySize})'
        return template_type.format(**self.data)

    def get_mangled_name(self):
        mangled_name = ''

        size_key = (self.data['secondarySize'] - 1) * 4 + self.data['primarySize'] - 1
        if size_key < 10:
            mangled_name += chr(ord('0') + size_key)
        else:
            mangled_name += chr(ord('A') + size_key - 10)

        mangled_name += get_basic_mangled_name(self.data['basic'])
        return mangled_name

    def get_human_readable_name(self):
        name = self.data['basic']
        name += str(self.data['primarySize'])
        if self.data['secondarySize'] > 1:
            name += 'x' + str(self.data['secondarySize'])
        return name

    def is_vector(self):
        return self.data['primarySize'] > 1 and self.data['secondarySize'] == 1

    def is_matrix(self):
        return self.data['secondarySize'] > 1

    def get_object_size(self):
        return self.data['primarySize'] * self.data['secondarySize']

    def specific_sampler_or_image_type(self, basic_type_prefix):
        if 'genType' in self.data:
            type = {}
            if 'basic' not in self.data:
                type['basic'] = {'': 'Float', 'I': 'Int', 'U': 'UInt'}[basic_type_prefix]
                type['primarySize'] = self.data['primarySize']
            else:
                type['basic'] = basic_type_prefix + self.data['basic']
                type['primarySize'] = 1
            type['precision'] = 'Undefined'
            return TType(type)
        return self

    def specific_type(self, vec_size):
        type = {}
        if 'genType' in self.data:
            type['basic'] = self.data['basic']
            type['precision'] = self.data['precision']
            type['qualifier'] = self.data['qualifier']
            type['primarySize'] = vec_size
            type['secondarySize'] = 1
            return TType(type)
        return self

    def parse_type(self, glsl_header_type):
        if glsl_header_type.startswith('out '):
            type_obj = self.parse_type(glsl_header_type[4:])
            type_obj['qualifier'] = 'Out'
            return type_obj
        if glsl_header_type.startswith('inout '):
            type_obj = self.parse_type(glsl_header_type[6:])
            type_obj['qualifier'] = 'InOut'
            return type_obj

        basic_type_map = {
            'float': 'Float',
            'int': 'Int',
            'uint': 'UInt',
            'bool': 'Bool',
            'void': 'Void',
            'atomic_uint': 'AtomicCounter',
            'yuvCscStandardEXT': 'YuvCscStandardEXT'
        }

        if glsl_header_type in basic_type_map:
            return {'basic': basic_type_map[glsl_header_type]}

        type_obj = {}

        basic_type_prefix_map = {'': 'Float', 'i': 'Int', 'u': 'UInt', 'b': 'Bool', 'v': 'Void'}

        vec_re = re.compile(r'^([iub]?)vec([234]?)$')
        vec_match = vec_re.match(glsl_header_type)
        if vec_match:
            type_obj['basic'] = basic_type_prefix_map[vec_match.group(1)]
            if vec_match.group(2) == '':
                # Type like "ivec" that represents either ivec2, ivec3 or ivec4
                type_obj['genType'] = 'vec'
            else:
                # vec with specific size
                type_obj['primarySize'] = int(vec_match.group(2))
            return type_obj

        mat_re = re.compile(r'^mat([234])(x([234]))?$')
        mat_match = mat_re.match(glsl_header_type)
        if mat_match:
            type_obj['basic'] = 'Float'
            if len(glsl_header_type) == 4:
                mat_size = int(mat_match.group(1))
                type_obj['primarySize'] = mat_size
                type_obj['secondarySize'] = mat_size
            else:
                type_obj['primarySize'] = int(mat_match.group(1))
                type_obj['secondarySize'] = int(mat_match.group(3))
            return type_obj

        gen_re = re.compile(r'^gen([IUB]?)Type$')
        gen_match = gen_re.match(glsl_header_type)
        if gen_match:
            type_obj['basic'] = basic_type_prefix_map[gen_match.group(1).lower()]
            type_obj['genType'] = 'yes'
            return type_obj

        if glsl_header_type.startswith('sampler'):
            type_obj['basic'] = glsl_header_type[0].upper() + glsl_header_type[1:]
            return type_obj

        if glsl_header_type.startswith('gsampler') or glsl_header_type.startswith('gimage'):
            type_obj['basic'] = glsl_header_type[1].upper() + glsl_header_type[2:]
            type_obj['genType'] = 'sampler_or_image'
            return type_obj

        if glsl_header_type == 'gvec4':
            return {'primarySize': 4, 'genType': 'sampler_or_image'}
        if glsl_header_type == 'gvec3':
            return {'primarySize': 3, 'genType': 'sampler_or_image'}

        raise Exception('Unrecognized type: ' + str(glsl_header_type))

def get_parsed_functions():

    def parse_function_parameters(parameters):
        if parameters == '':
            return []
        parametersOut = []
        parameters = parameters.split(', ')
        for parameter in parameters:
            parametersOut.append(TType(parameter.strip()))
        return parametersOut

    lines = []
    with open(functions_txt_filename) as f:
        lines = f.readlines()
    lines = [line.strip() for line in lines if line.strip() != '' and not line.strip().startswith('//')]

    fun_re = re.compile(r'^(\w+) (\w+)\((.*)\);$')

    parsed_functions = OrderedDict()
    group_stack = []
    default_metadata = {}

    for line in lines:
        fun_match = fun_re.match(line)
        if line.startswith('GROUP BEGIN '):
            group_rest = line[12:].strip()
            group_parts = group_rest.split(' ', 1)
            current_group = {
                'functions': [],
                'name': group_parts[0],
                'subgroups': {}
            }
            if len(group_parts) > 1:
                group_metadata = json.loads(group_parts[1])
                current_group.update(group_metadata)
            group_stack.append(current_group)
        elif line.startswith('GROUP END '):
            group_end_name = line[10:].strip()
            current_group = group_stack[-1]
            if current_group['name'] != group_end_name:
                raise Exception('GROUP END: Unexpected function group name "' + group_end_name + '" was expecting "' + current_group['name'] + '"')
            group_stack.pop()
            is_top_level_group = (len(group_stack) == 0)
            if is_top_level_group:
                parsed_functions[current_group['name']] = current_group
                default_metadata = {}
            else:
                super_group = group_stack[-1]
                super_group['subgroups'][current_group['name']] = current_group
        elif line.startswith('DEFAULT METADATA'):
            line_rest = line[16:].strip()
            default_metadata = json.loads(line_rest)
        elif fun_match:
            return_type = fun_match.group(1)
            name = fun_match.group(2)
            parameters = fun_match.group(3)
            function_props = {
                'name': name,
                'returnType': TType(return_type),
                'parameters': parse_function_parameters(parameters)
            }
            function_props.update(default_metadata)
            group_stack[-1]['functions'].append(function_props)
        else:
            raise Exception('Unexpected function input line: ' + line)

    return parsed_functions

parsed_functions = get_parsed_functions()

if args.dump_intermediate_json:
    with open('builtin_functions.json', 'w') as outfile:
        def serialize_obj(obj):
            if isinstance(obj, TType):
                return obj.data
            else:
                raise "Cannot serialize to JSON: " + str(obj)
        json.dump(parsed_functions, outfile, indent=4, separators=(',', ': '), default=serialize_obj)

with open(variables_json_filename) as f:
    parsed_variables = json.load(f, object_pairs_hook=OrderedDict)

# Declarations of symbol unique ids
builtin_id_declarations = []

# Definitions of symbol unique ids needed for those ids used outside of constexpr expressions.
builtin_id_definitions = []

# Declarations of name string variables
name_declarations = set()

# Declarations of builtin TVariables
variable_declarations = []

# Declarations of parameter arrays for builtin TFunctions. Map from C++ variable name to the full
# declaration.
parameter_declarations = {}

# Declarations of builtin TFunctions
function_declarations = []

# Functions for querying the pointer to a specific TVariable.
get_variable_declarations = []
get_variable_definitions = []

# Code for defining TVariables stored as members of TSymbolTable.
declare_member_variables = []
init_member_variables = []

# Code for querying builtins.
get_builtin_if_statements = GroupedList()

# Declarations of UnmangledBuiltIn objects
unmangled_builtin_declarations = set()

# Code for querying builtin function unmangled names.
unmangled_function_if_statements = GroupedList()

# Code for testing that script-generated hashes match with runtime computed hashes.
script_generated_hash_tests = OrderedDict()

# Functions for testing whether a builtin belongs in group.
is_in_group_definitions = []

# Counts of variables with a certain name string:
variable_name_count = {}

id_counter = 0

fnvPrime = 16777619
def hash32(str):
    fnvOffsetBasis = 0x811c9dc5
    hash = fnvOffsetBasis
    for c in str:
        hash = hash ^ ord(c)
        hash = (hash * fnvPrime) & 0xffffffff
    return hash

def mangledNameHash(str, save_test = True):
    hash = hash32(str)
    index = 0
    max_six_bit_value = (1 << 6) - 1
    paren_location = max_six_bit_value
    has_array_or_block_param_bit = 0
    for c in str:
        if c == '(':
            paren_location = index
        elif c == '{' or c == '[':
            has_array_or_block_param_bit = 1
        index += 1
    hash = ((hash >> 13) ^ (hash & 0x1fff)) | (index << 19) | (paren_location << 25) | (has_array_or_block_param_bit << 31)
    if save_test:
        sanity_check = '    ASSERT_EQ(0x{hash}u, ImmutableString("{str}").mangledNameHash());'.format(hash = ('%08x' % hash), str = str)
        script_generated_hash_tests.update({sanity_check: None})
    return hash

def get_suffix(props):
    if 'suffix' in props:
        return props['suffix']
    return ''

def get_extension(props):
    if 'extension' in props:
        return props['extension']
    return 'UNDEFINED'

def get_op(name, function_props):
    if 'op' not in function_props:
        raise Exception('function op not defined')
    if function_props['op'] == 'auto':
        return name[0].upper() + name[1:]
    return function_props['op']

def get_known_to_not_have_side_effects(function_props):
    if 'op' in function_props and function_props['op'] != 'CallBuiltInFunction':
        if 'hasSideEffects' in function_props:
            return 'false'
        else:
            for param in get_parameters(function_props):
                if 'qualifier' in param.data and (param.data['qualifier'] == 'Out' or param.data['qualifier'] == 'InOut'):
                    return 'false'
            return 'true'
    return 'false'

def get_parameters(function_props):
    if 'parameters' in function_props:
        return function_props['parameters']
    return []

def get_function_mangled_name(function_name, parameters):
    mangled_name = function_name + '('
    for param in parameters:
        mangled_name += param.get_mangled_name()
    return mangled_name

def get_function_human_readable_name(function_name, parameters):
    name = function_name
    for param in parameters:
        name += '_' + param.get_human_readable_name()
    return name

ttype_mangled_name_variants = []
for basic_type in basic_types_enumeration:
    primary_sizes = [1]
    secondary_sizes = [1]
    if basic_type in ['Float', 'Int', 'UInt', 'Bool']:
        primary_sizes = [1, 2, 3, 4]
    if basic_type == 'Float':
        secondary_sizes = [1, 2, 3, 4]
    for primary_size in primary_sizes:
        for secondary_size in secondary_sizes:
            type = TType({'basic': basic_type, 'primarySize': primary_size, 'secondarySize': secondary_size})
            ttype_mangled_name_variants.append(type.get_mangled_name())

def gen_parameters_variant_ids(str_len):
    # Note that this doesn't generate variants with array parameters or struct / interface block parameters. They are assumed to have been filtered out separately.
    if str_len % 2 != 0:
        raise Exception('Expecting parameters mangled name length to be divisible by two')
    num_variants = pow(len(ttype_mangled_name_variants), str_len / 2)
    return xrange(num_variants)

def get_parameters_mangled_name_variant(variant_id, paren_location, total_length):
    str_len = total_length - paren_location - 1
    if str_len % 2 != 0:
        raise Exception('Expecting parameters mangled name length to be divisible by two')
    variant = ''
    while (len(variant)) < str_len:
        parameter_index = len(variant) / 2
        parameter_variant_index = variant_id
        for i in xrange(parameter_index):
            parameter_variant_index = parameter_variant_index / len(ttype_mangled_name_variants)
        parameter_variant_index = parameter_variant_index % len(ttype_mangled_name_variants)
        variant += ttype_mangled_name_variants[parameter_variant_index]
    return variant

# Calculate the mangled name hash of a common prefix string that's been pre-hashed with hash32()
# plus a variant of the parameters. This is faster than constructing the whole string and then
# calculating the hash for that.
num_type_variants = len(ttype_mangled_name_variants)
def get_mangled_name_variant_hash(prefix_hash32, variant_id, paren_location, total_length):
    hash = prefix_hash32
    parameter_count = (total_length - paren_location) >> 1
    parameter_variant_id_base = variant_id
    for parameter_index in xrange(parameter_count):
        parameter_variant_index = parameter_variant_id_base % num_type_variants
        param_str = ttype_mangled_name_variants[parameter_variant_index]
        hash = hash ^ ord(param_str[0])
        hash = (hash * fnvPrime) & 0xffffffff
        hash = hash ^ ord(param_str[1])
        hash = (hash * fnvPrime) & 0xffffffff
        parameter_variant_id_base = parameter_variant_id_base / num_type_variants
    return ((hash >> 13) ^ (hash & 0x1fff)) | (total_length << 19) | (paren_location << 25)

# Sanity check for get_mangled_name_variant_hash:
if get_mangled_name_variant_hash(hash32("atan("), 3, 4, len("atan(0123")) != mangledNameHash("atan(" + get_parameters_mangled_name_variant(3, 4, len("atan(0123"))):
    raise Exception("get_mangled_name_variant_hash sanity check failed")

def mangled_name_hash_can_collide_with_different_parameters(function_variant_props):
    # We exhaustively search through all possible lists of parameters and see if any other mangled
    # name has the same hash.
    mangled_name = function_variant_props['mangled_name']
    mangled_name_len = len(mangled_name)
    hash = mangledNameHash(mangled_name)
    mangled_name_prefix = function_variant_props['name'] + '('
    paren_location = len(mangled_name_prefix) - 1
    prefix_hash32 = hash32(mangled_name_prefix)
    parameters_mangled_name_len = len(mangled_name) - len(mangled_name_prefix)
    parameters_mangled_name = mangled_name[len(mangled_name_prefix):]
    if (parameters_mangled_name_len > 6):
        # This increases the complexity of searching for hash collisions considerably, so rather than doing it we just conservatively assume that a hash collision may be possible.
        return True
    for variant_id in gen_parameters_variant_ids(parameters_mangled_name_len):
        if get_mangled_name_variant_hash(prefix_hash32, variant_id, paren_location, mangled_name_len) == hash and get_parameters_mangled_name_variant(variant_id, paren_location, mangled_name_len) != parameters_mangled_name:
            return True
    return False

def get_unique_identifier_name(function_name, parameters):
    unique_name = function_name + '_'
    for param in parameters:
        unique_name += param.get_mangled_name()
    return unique_name

def get_variable_name_to_store_parameter(param):
    unique_name = 'pt'
    if 'qualifier' in param.data:
        if param.data['qualifier'] == 'Out':
            unique_name += '_o_'
        if param.data['qualifier'] == 'InOut':
            unique_name += '_io_'
    unique_name += param.get_mangled_name()
    return unique_name

def get_variable_name_to_store_parameters(parameters):
    if len(parameters) == 0:
        return 'empty'
    unique_name = 'p'
    for param in parameters:
        if 'qualifier' in param.data:
            if param.data['qualifier'] == 'Out':
                unique_name += '_o_'
            if param.data['qualifier'] == 'InOut':
                unique_name += '_io_'
        unique_name += param.get_mangled_name()
    return unique_name

def define_constexpr_variable(template_args):
    template_variable_declaration = 'constexpr const TVariable kVar_{name_with_suffix}(BuiltInId::{name_with_suffix}, BuiltInName::{name}, SymbolType::BuiltIn, TExtension::{extension}, {type});'
    variable_declarations.append(template_variable_declaration.format(**template_args))

def gen_function_variants(function_name, function_props):
    function_variants = []
    parameters = get_parameters(function_props)
    function_is_gen_type = False
    gen_type = set()
    for param in parameters:
        if 'genType' in param.data:
            if param.data['genType'] not in ['sampler_or_image', 'vec', 'yes']:
                raise Exception('Unexpected value of genType "' + str(param.data['genType']) + '" should be "sampler_or_image", "vec", or "yes"')
            gen_type.add(param.data['genType'])
            if len(gen_type) > 1:
                raise Exception('Unexpected multiple values of genType set on the same function: ' + str(list(gen_type)))
    if len(gen_type) == 0:
        function_variants.append(function_props)
        return function_variants

    # If we have a gsampler_or_image then we're generating variants for float, int and uint
    # samplers.
    if 'sampler_or_image' in gen_type:
        types = ['', 'I', 'U']
        for type in types:
            variant_props = function_props.copy()
            variant_parameters = []
            for param in parameters:
                variant_parameters.append(param.specific_sampler_or_image_type(type))
            variant_props['parameters'] = variant_parameters
            variant_props['returnType'] = function_props['returnType'].specific_sampler_or_image_type(type)
            function_variants.append(variant_props)
        return function_variants

    # If we have a normal gentype then we're generating variants for different sizes of vectors.
    sizes = range(1, 5)
    if 'vec' in gen_type:
        sizes = range(2, 5)
    for size in sizes:
        variant_props = function_props.copy()
        variant_parameters = []
        for param in parameters:
            variant_parameters.append(param.specific_type(size))
        variant_props['parameters'] = variant_parameters
        variant_props['returnType'] = function_props['returnType'].specific_type(size)
        function_variants.append(variant_props)
    return function_variants

defined_function_variants = set()
defined_parameter_names = set()

def process_single_function_group(condition, group_name, group):
    global id_counter
    if 'functions' not in group:
        return

    for function_props in group['functions']:
        function_name = function_props['name']
        level = function_props['level']
        extension = get_extension(function_props)
        template_args = {
            'name': function_name,
            'name_with_suffix': function_name + get_suffix(function_props),
            'level': level,
            'extension': extension,
            'op': get_op(function_name, function_props),
            'known_to_not_have_side_effects': get_known_to_not_have_side_effects(function_props)
        }

        function_variants = gen_function_variants(function_name, function_props)

        template_name_declaration = 'constexpr const ImmutableString {name_with_suffix}("{name}");'
        name_declaration = template_name_declaration.format(**template_args)
        if not name_declaration in name_declarations:
            name_declarations.add(name_declaration)

        template_unmangled_if = """if (name == BuiltInName::{name_with_suffix})
{{
    return &UnmangledBuiltIns::{extension};
}}"""
        unmangled_if = template_unmangled_if.format(**template_args)
        unmangled_builtin_no_condition = unmangled_function_if_statements.get(level, 'NO_CONDITION', function_name)
        if unmangled_builtin_no_condition != None and unmangled_builtin_no_condition['extension'] == 'UNDEFINED':
            # We already have this unmangled name without a condition nor extension on the same level. No need to add a duplicate with a condition.
            pass
        elif (not unmangled_function_if_statements.has_key(level, condition, function_name)) or extension == 'UNDEFINED':
            # We don't have this unmangled builtin recorded yet or we might replace an unmangled builtin from an extension with one from core.
            unmangled_function_if_statements.add_obj(level, condition, function_name, {'hash_matched_code': unmangled_if, 'extension': extension})
            unmangled_builtin_declarations.add('constexpr const UnmangledBuiltIn {extension}(TExtension::{extension});'.format(**template_args))

        for function_props in function_variants:
            template_args['id'] = id_counter

            parameters = get_parameters(function_props)

            template_args['unique_name'] = get_unique_identifier_name(template_args['name_with_suffix'], parameters)

            if template_args['unique_name'] in defined_function_variants:
                continue
            defined_function_variants.add(template_args['unique_name'])

            template_args['param_count'] = len(parameters)
            template_args['return_type'] = function_props['returnType'].get_statictype_string()
            template_args['mangled_name'] = get_function_mangled_name(function_name, parameters)
            template_args['human_readable_name'] = get_function_human_readable_name(template_args['name_with_suffix'], parameters)
            template_args['mangled_name_length'] = len(template_args['mangled_name'])

            template_builtin_id_declaration = '    static constexpr const TSymbolUniqueId {human_readable_name} = TSymbolUniqueId({id});'
            builtin_id_declarations.append(template_builtin_id_declaration.format(**template_args))
            template_builtin_id_definition = 'constexpr const TSymbolUniqueId BuiltInId::{human_readable_name};'
            builtin_id_definitions.append(template_builtin_id_definition.format(**template_args))

            parameters_list = []
            for param in parameters:
                unique_param_name = get_variable_name_to_store_parameter(param)
                param_template_args = {
                    'name': '_empty',
                    'name_with_suffix': unique_param_name,
                    'type': param.get_statictype_string(),
                    'extension': 'UNDEFINED'
                }
                if unique_param_name not in defined_parameter_names:
                    id_counter += 1
                    param_template_args['id'] = id_counter
                    template_builtin_id_declaration = '    static constexpr const TSymbolUniqueId {name_with_suffix} = TSymbolUniqueId({id});'
                    builtin_id_declarations.append(template_builtin_id_declaration.format(**param_template_args))
                    define_constexpr_variable(param_template_args)
                    defined_parameter_names.add(unique_param_name)
                parameters_list.append('&BuiltInVariable::kVar_{name_with_suffix}'.format(**param_template_args));

            template_args['parameters_var_name'] = get_variable_name_to_store_parameters(parameters)
            if len(parameters) > 0:
                template_args['parameters_list'] = ', '.join(parameters_list)
                template_parameter_list_declaration = 'constexpr const TVariable *{parameters_var_name}[{param_count}] = {{ {parameters_list} }};'
                parameter_declarations[template_args['parameters_var_name']] = template_parameter_list_declaration.format(**template_args)
            else:
                template_parameter_list_declaration = 'constexpr const TVariable **{parameters_var_name} = nullptr;'
                parameter_declarations[template_args['parameters_var_name']] = template_parameter_list_declaration.format(**template_args)

            template_function_declaration = 'constexpr const TFunction kFunction_{unique_name}(BuiltInId::{human_readable_name}, BuiltInName::{name_with_suffix}, TExtension::{extension}, BuiltInParameters::{parameters_var_name}, {param_count}, {return_type}, EOp{op}, {known_to_not_have_side_effects});'
            function_declarations.append(template_function_declaration.format(**template_args))

            # If we can make sure that there's no other mangled name with the same length, function
            # name and hash, then we can only check the mangled name length and the function name
            # instead of checking the whole mangled name.
            template_mangled_if = ''
            if mangled_name_hash_can_collide_with_different_parameters(template_args):
                template_mangled_name_declaration = 'constexpr const ImmutableString {unique_name}("{mangled_name}");'
                name_declarations.add(template_mangled_name_declaration.format(**template_args))
                template_mangled_if = """if (name == BuiltInName::{unique_name})
{{
    return &BuiltInFunction::kFunction_{unique_name};
}}"""
            else:
                template_mangled_if = """if (name.beginsWith(BuiltInName::{name_with_suffix}))
{{
    ASSERT(name.length() == {mangled_name_length});
    return &BuiltInFunction::kFunction_{unique_name};
}}"""
            mangled_if = template_mangled_if.format(**template_args)
            get_builtin_if_statements.add_obj(level, condition, template_args['mangled_name'], {'hash_matched_code': mangled_if})

            id_counter += 1

def process_function_group(group_name, group):
    global id_counter
    first_id = id_counter

    condition = 'NO_CONDITION'
    if 'condition' in group:
        condition = group['condition']

    process_single_function_group(condition, group_name, group)

    if 'subgroups' in group:
        for subgroup_name, subgroup in group['subgroups'].iteritems():
            process_function_group(group_name + subgroup_name, subgroup)

    if 'queryFunction' in group:
        template_args = {
            'first_id': first_id,
            'last_id': id_counter - 1,
            'group_name': group_name
        }
        template_is_in_group_definition = """bool is{group_name}(const TFunction *func)
{{
    int id = func->uniqueId().get();
    return id >= {first_id} && id <= {last_id};
}}"""
        is_in_group_definitions.append(template_is_in_group_definition.format(**template_args))

for group_name, group in parsed_functions.iteritems():
    process_function_group(group_name, group)

def prune_parameters_arrays():
    # We can share parameters arrays between functions in case one array is a subarray of another.
    global parameter_declarations
    parameter_variable_name_replacements = {}
    used_param_variable_names = set()
    for param_variable_name, param_declaration in sorted(parameter_declarations.iteritems(), key=lambda item: -len(item[0])):
        replaced = False
        for used in used_param_variable_names:
            if used.startswith(param_variable_name):
                parameter_variable_name_replacements[param_variable_name] = used
                replaced = True
                break
        if not replaced:
            used_param_variable_names.add(param_variable_name)
    parameter_declarations = [value for key, value in parameter_declarations.iteritems() if key in used_param_variable_names]

    for i in xrange(len(function_declarations)):
        for replaced, replacement in parameter_variable_name_replacements.iteritems():
            function_declarations[i] = function_declarations[i].replace('BuiltInParameters::' + replaced + ',', 'BuiltInParameters::' + replacement + ',')
prune_parameters_arrays()

def process_single_variable_group(condition, group_name, group):
    global id_counter
    if 'variables' not in group:
        return
    for variable_name, props in group['variables'].iteritems():
        level = props['level']
        template_args = {
            'id': id_counter,
            'name': variable_name,
            'name_with_suffix': variable_name + get_suffix(props),
            'level': props['level'],
            'extension': get_extension(props),
            'class': 'TVariable'
        }

        template_builtin_id_declaration = '    static constexpr const TSymbolUniqueId {name_with_suffix} = TSymbolUniqueId({id});'
        builtin_id_declarations.append(template_builtin_id_declaration.format(**template_args))
        template_builtin_id_definition = 'constexpr const TSymbolUniqueId BuiltInId::{name_with_suffix};'
        builtin_id_definitions.append(template_builtin_id_definition.format(**template_args))

        template_name_declaration = 'constexpr const ImmutableString {name}("{name}");'
        name_declarations.add(template_name_declaration.format(**template_args))

        is_member = True
        template_init_variable = ''

        if 'type' in props:
            if props['type']['basic'] != 'Bool' and 'precision' not in props['type']:
                raise Exception('Missing precision for variable ' + variable_name)
            template_args['type'] = TType(props['type']).get_statictype_string()

        if 'fields' in props:
            # Handle struct and interface block definitions.
            template_args['class'] = props['class']
            template_args['fields'] = 'fields_{name_with_suffix}'.format(**template_args)
            init_member_variables.append('    TFieldList *{fields} = new TFieldList();'.format(**template_args))
            for field_name, field_type in props['fields'].iteritems():
                template_args['field_name'] = field_name
                template_args['field_type'] = TType(field_type).get_dynamic_type_string()
                template_name_declaration = 'constexpr const ImmutableString {field_name}("{field_name}");'
                name_declarations.add(template_name_declaration.format(**template_args))
                template_add_field = '    {fields}->push_back(new TField({field_type}, BuiltInName::{field_name}, zeroSourceLoc, SymbolType::BuiltIn));'
                init_member_variables.append(template_add_field.format(**template_args))
            template_init_temp_variable = '    {class} *{name_with_suffix} = new {class}(BuiltInId::{name_with_suffix}, BuiltInName::{name}, TExtension::{extension}, {fields});'
            init_member_variables.append(template_init_temp_variable.format(**template_args))
            if 'private' in props and props['private']:
                is_member = False
            else:
                template_init_variable = '    mVar_{name_with_suffix} = {name_with_suffix};'

        elif 'initDynamicType' in props:
            # Handle variables whose type can't be expressed as TStaticType
            # (type is a struct or has variable array size for example).
            template_args['type_name'] = 'type_{name_with_suffix}'.format(**template_args)
            template_args['type'] = template_args['type_name']
            template_args['initDynamicType'] = props['initDynamicType'].format(**template_args)
            template_init_variable = """    {initDynamicType}
    {type_name}->realize();
    mVar_{name_with_suffix} = new TVariable(BuiltInId::{name_with_suffix}, BuiltInName::{name}, SymbolType::BuiltIn, TExtension::{extension}, {type});"""

        elif 'value' in props:
            # Handle variables with constant value, such as gl_MaxDrawBuffers.
            if props['value'] != 'resources':
                raise Exception('Unrecognized value source in variable properties: ' + str(props['value']))
            resources_key = variable_name[3:]
            if 'valueKey' in props:
                resources_key = props['valueKey']
            template_args['value'] = 'resources.' + resources_key
            template_args['object_size'] = TType(props['type']).get_object_size()
            template_init_variable = """    mVar_{name_with_suffix} = new TVariable(BuiltInId::{name_with_suffix}, BuiltInName::{name}, SymbolType::BuiltIn, TExtension::{extension}, {type});
    {{
        TConstantUnion *unionArray = new TConstantUnion[{object_size}];
        unionArray[0].setIConst({value});
        mVar_{name_with_suffix}->shareConstPointer(unionArray);
    }}"""
            if template_args['object_size'] > 1:
                template_init_variable = """    mVar_{name_with_suffix} = new TVariable(BuiltInId::{name_with_suffix}, BuiltInName::{name}, SymbolType::BuiltIn, TExtension::{extension}, {type});
    {{
        TConstantUnion *unionArray = new TConstantUnion[{object_size}];
        for (size_t index = 0u; index < {object_size}; ++index)
        {{
            unionArray[index].setIConst({value}[index]);
        }}
        mVar_{name_with_suffix}->shareConstPointer(unionArray);
    }}"""

        else:
            # Handle variables that can be stored as constexpr TVariable like
            # gl_Position, gl_FragColor etc.
            define_constexpr_variable(template_args)
            is_member = False

            template_get_variable_declaration = 'const TVariable *{name_with_suffix}();'
            get_variable_declarations.append(template_get_variable_declaration.format(**template_args))

            template_get_variable_definition = """const TVariable *{name_with_suffix}()
{{
    return &kVar_{name_with_suffix};
}}
"""
            get_variable_definitions.append(template_get_variable_definition.format(**template_args))

            if level != 'GLSL_BUILTINS':
                template_name_if = """if (name == BuiltInName::{name})
{{
    return &BuiltInVariable::kVar_{name_with_suffix};
}}"""
                name_if = template_name_if.format(**template_args)
                get_builtin_if_statements.add_obj(level, condition, template_args['name'], {'hash_matched_code': name_if})

        if is_member:
            get_condition = condition
            init_conditionally = (condition != 'NO_CONDITION' and variable_name_count[variable_name] == 1)
            if init_conditionally:
                # Instead of having the condition if statement at lookup, it's cheaper to have it at initialization time.
                init_member_variables.append('    if ({condition})\n    {{'.format(condition = condition))
                template_args['condition_comment'] = '\n    // Only initialized if {condition}'.format(condition = condition)
                get_condition = 'NO_CONDITION'
            else:
                template_args['condition_comment'] = ''
            init_member_variables.append(template_init_variable.format(**template_args))
            if init_conditionally:
                init_member_variables.append('    }')

            template_declare_member_variable = '{class} *mVar_{name_with_suffix} = nullptr;'
            declare_member_variables.append(template_declare_member_variable.format(**template_args))

            if level != 'GLSL_BUILTINS':
                template_name_if = """if (name == BuiltInName::{name})
{{{condition_comment}
    return mVar_{name_with_suffix};
}}"""
                name_if = template_name_if.format(**template_args)
                get_builtin_if_statements.add_obj(level, get_condition, variable_name, {'hash_matched_code': name_if})

        id_counter += 1

def count_variable_names(group):
    if 'variables' in group:
        for name in group['variables'].iterkeys():
            if name not in variable_name_count:
                variable_name_count[name] = 1
            else:
                variable_name_count[name] += 1
    if 'subgroups' in group:
        for subgroup_name, subgroup in group['subgroups'].iteritems():
            count_variable_names(subgroup)

def process_variable_group(parent_condition, group_name, group):
    condition = 'NO_CONDITION'
    if 'condition' in group:
        condition = group['condition']

    if parent_condition != 'NO_CONDITION':
        if condition == 'NO_CONDITION':
            condition = parent_condition
        else:
            condition = '({cond1}) && ({cond2})'.format(cond1 = parent_condition, cond2 = condition)

    process_single_variable_group(condition, group_name, group)

    if 'subgroups' in group:
        for subgroup_name, subgroup in group['subgroups'].iteritems():
            process_variable_group(condition, subgroup_name, subgroup)

for group_name, group in parsed_variables.iteritems():
    count_variable_names(group)

for group_name, group in parsed_variables.iteritems():
    process_variable_group('NO_CONDITION', group_name, group)

output_strings = {
    'script_name': os.path.basename(__file__),
    'copyright_year': date.today().year,

    'builtin_id_declarations': '\n'.join(builtin_id_declarations),
    'builtin_id_definitions': '\n'.join(builtin_id_definitions),
    'last_builtin_id': id_counter - 1,
    'name_declarations': '\n'.join(sorted(list(name_declarations))),

    'function_data_source_name': functions_txt_filename,
    'function_declarations': '\n'.join(function_declarations),
    'parameter_declarations': '\n'.join(sorted(parameter_declarations)),

    'is_in_group_definitions': '\n'.join(is_in_group_definitions),

    'variable_data_source_name': variables_json_filename,
    'variable_declarations': '\n'.join(sorted(variable_declarations)),
    'get_variable_declarations': '\n'.join(sorted(get_variable_declarations)),
    'get_variable_definitions': '\n'.join(sorted(get_variable_definitions)),
    'unmangled_builtin_declarations': '\n'.join(sorted(unmangled_builtin_declarations)),

    'declare_member_variables': '\n'.join(declare_member_variables),
    'init_member_variables': '\n'.join(init_member_variables),

    'get_unmangled_builtin': unmangled_function_if_statements.get_switch_code(),
    'get_builtin': get_builtin_if_statements.get_switch_code(),
    'max_unmangled_name_length': unmangled_function_if_statements.get_max_name_length(),
    'max_mangled_name_length': get_builtin_if_statements.get_max_name_length(),

    'script_generated_hash_tests': '\n'.join(script_generated_hash_tests.iterkeys())
}

with open('../../tests/compiler_tests/ImmutableString_test_autogen.cpp', 'wt') as outfile_cpp:
    output_cpp = template_immutablestringtest_cpp.format(**output_strings)
    outfile_cpp.write(output_cpp)

with open('tree_util/BuiltIn_autogen.h', 'wt') as outfile_header:
    output_header = template_builtin_header.format(**output_strings)
    outfile_header.write(output_header)

with open('SymbolTable_autogen.cpp', 'wt') as outfile_cpp:
    output_cpp = template_symboltable_cpp.format(**output_strings)
    outfile_cpp.write(output_cpp)

with open('ParseContext_autogen.h', 'wt') as outfile_header:
    output_header = template_parsecontext_header.format(**output_strings)
    outfile_header.write(output_header)

with open('SymbolTable_autogen.h', 'wt') as outfile_h:
    output_h = template_symboltable_h.format(**output_strings)
    outfile_h.write(output_h)

with open(hash_filename, 'wt') as hash_file:
    hash_file.write(input_hash)
kc3-lang/angle/src/compiler/translator/gen_builtin_symbols.py

Commit

kc3-lang/angle /src/compiler/translator/gen_builtin_symbols.py
