gen_vk_gl_cts_build.py

Branch :

Show log
Commit
Author : Tim Van Patten
Date : 2019-10-25 16:50:01
Hash : 11c487a8
Message : Manually copy dEQP data files to output directory dEQP has data files that live in several directories within dEQP's source tree. For example, GLES3 has data files that live within: data/gles3/data/ external/graphicsfuzz/data/gles3/ However, we can only tell dEQP about a single data directory during initialization of dEQP. To get around this, we are manually copying all of the necessary data files to the output generated files directory and pointing dEQP to this single directory. This also helps us solve a second problem related to the paths that dEQP uses when accessing graphicsfuzz data files. For the graphicsfuzz tests, dEQP will attempt to open the necessary shaders by accessing them with the path: data/gles3/graphicsfuzz/ However, those files would normally live at the path that matches their location within the source tree: external/graphicsfuzz/data/gles3/graphicsfuzz/ As part of the manual copy of these data files, we are also able to strip the extra 'external/graphicsfuzz/' portion of the path. Bug: angleproject:2322 Test: dEQP Change-Id: Ibc96442c221485e2f246890fa8fe51f090c5e222 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1881759 Commit-Queue: Tim Van Patten <timvp@google.com> Reviewed-by: Tim Van Patten <timvp@google.com>

scripts/gen_vk_gl_cts_build.py

#  Copyright 2019 The ANGLE Project Authors. All rights reserved.
#  Use of this source code is governed by a BSD-style license that can be
#  found in the LICENSE file.
#
#  Code generation for:
#  - src/tests/deqp_support/BUILD.gn
#  - src/tests/deqp_data.gni
#  NOTE: don't run this script directly. Run scripts/run_code_generation.py.

import errno
import os
import sys
import shutil


def initDataDirectories(dataDirectories):
    dataDirectories.append(os.path.join("data", "gles2"))
    dataDirectories.append(os.path.join("data", "gles3"))
    dataDirectories.append(os.path.join("data", "gles31"))
    dataDirectories.append(os.path.join("external", "graphicsfuzz", "data", "gles3"))
    dataDirectories.append(os.path.join("external", "openglcts", "data", "gles3"))


def initPathReplacements(pathReplacements):
    # The GraphicsFuzz data files need the 'external/graphicsfuzz/' prefix removed
    pathToReplace = os.path.join("external", "graphicsfuzz", "")  # Include trailing slash
    pathReplacements[pathToReplace] = ""
    # The KHR dEQP tests expect a root prefix of "gl_cts" for some reason.
    pathToReplace = os.path.join("external", "openglcts", "")  # Include trailing slash
    pathReplacements[pathToReplace] = os.path.join("data", "gl_cts", "")


def createBuildGnFile(buildGnPath):
    # Cleanup the old file
    if os.path.exists(buildGnPath):
        os.remove(buildGnPath)
    # Make the new one
    return open(buildGnPath, "w+")


def createGniFile(gniFilename):
    # Cleanup the old file
    if os.path.exists(gniFilename):
        os.remove(gniFilename)
    # Make the new one
    return open(gniFilename, "w+")


def writeFileHeader(fileIn):
    templateFileHeader = """# GENERATED FILE - DO NOT EDIT.
# Generated by: {script_name}
#
# Copyright 2019 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""

    fileHeader = templateFileHeader.format(script_name=os.path.basename(__file__))
    fileIn.write(fileHeader)


def fixDestinationDirectory(pathReplacements, relativeDirectory):
    for pathReplacement in pathReplacements:
        if pathReplacement in relativeDirectory:
            return relativeDirectory.replace(pathReplacement, pathReplacements[pathReplacement])
    return relativeDirectory


def convertPathToVarName(path):
    return path.replace(os.sep, "_")


def getCMakeLists(deqpSrcDir):
    cmakeLists = []
    for root, directories, filenames in os.walk(deqpSrcDir):
        for filename in filenames:
            relativeDirectory = os.path.relpath(root, deqpSrcDir)
            if filename == "CMakeLists.txt":
                cmakeLists.append(os.path.join(relativeDirectory, filename))
    return cmakeLists


def main():
    # List of directories containing data files
    dataDirectories = []
    # List of directories to exclude from the copy
    excludedDirectories = [
        ".git",
    ]
    # List of files to exclude from the copy
    excludedFilenames = [
        "LICENSE",
    ]
    # Dictionary of parts of paths that need to be replaced
    # Key: Part of path to be replaced
    # Value: What to replace it with
    pathReplacements = {}
    # List of unique relative directories for the copy() command outputs
    relativeDirectories = []
    # VK-GL-CTS source directory
    deqpSourceDirectory = os.path.join("..", "third_party", "VK-GL-CTS", "src")
    # Tests Directory
    testsDirectory = os.path.join("..", "src", "tests")
    # dEQP Support Directory
    deqpSupportDirectory = "deqp_support"
    # BUILD.gn file to write to
    buildGnFilename = "BUILD.gn"
    # Path to BUILD.gn
    buildGnPath = os.path.join(testsDirectory, deqpSupportDirectory, buildGnFilename)
    # dEQP data GNI File to write to
    dataGniFilename = os.path.join(testsDirectory, deqpSupportDirectory, "deqp_data_autogen.gni")

    # run_code_generation.py parameters.
    if len(sys.argv) > 1:
        # All CMakeLists.txt in the dEQP source tree (at the time)
        cmakeDirs = getCMakeLists(deqpSourceDirectory)
        inputs = [os.path.join(deqpSourceDirectory, "%s" % dir) for dir in cmakeDirs]
        outputs = [dataGniFilename, buildGnPath]

        if sys.argv[1] == 'inputs':
            print(','.join(inputs))
        elif sys.argv[1] == 'outputs':
            print(','.join(outputs))
        else:
            print('Invalid script parameters')
            return 1
        return 0

    deqpSrcDir = os.path.abspath(os.path.join(sys.path[0], deqpSourceDirectory))

    initDataDirectories(dataDirectories)
    initPathReplacements(pathReplacements)

    dataFiles = []
    for dataDir in dataDirectories:
        dataPath = os.path.join(deqpSrcDir, dataDir)
        for root, directories, filenames in os.walk(dataPath):
            for filename in filenames:
                relativeDirectory = os.path.relpath(root, deqpSrcDir)
                # Skip any excluded directories
                if any(directory in relativeDirectory for directory in excludedDirectories):
                    continue
                # Skip any excluded files
                if any(excludedFilename in filename for excludedFilename in excludedFilenames):
                    continue
                # Record the relative directories and full paths to each data file
                if relativeDirectory not in relativeDirectories:
                    relativeDirectories.append(relativeDirectory)
                dataFiles.append(os.path.join(relativeDirectory, filename))

    dataFiles.sort()
    relativeDirectories.sort()

    #
    # BUILD.gn
    #
    buildGnFile = createBuildGnFile(buildGnPath)
    writeFileHeader(buildGnFile)
    # Definitions
    buildGnFile.write("deqp_path = \"../../../third_party/VK-GL-CTS/src\"\n")
    # Create the copy() commands
    templateFilesToCopy = """    "$deqp_path/{dataFile}",
"""
    templateCopyCommand = """
copy("vk_gl_cts_data_{relDir}") {{
  sources = [
    {filesToCopy}
  ]
  outputs = [ "$root_gen_dir/vk_gl_cts_data/{destDir}/{{{{source_file_part}}}}" ]
}}
"""
    for relativeDirectory in relativeDirectories:
        filesToCopy = ""
        for dataFile in dataFiles:
            path, filename = os.path.split(dataFile)
            if relativeDirectory == path:
                filesToCopy += templateFilesToCopy.format(dataFile=dataFile)
        copyCommand = ""
        destDir = fixDestinationDirectory(pathReplacements, relativeDirectory)
        copyCommand += templateCopyCommand.format(
            relDir=convertPathToVarName(relativeDirectory),
            filesToCopy=filesToCopy,
            destDir=destDir)
        buildGnFile.write(copyCommand)

    #
    # .gni
    #
    gniFile = createGniFile(dataGniFilename)
    writeFileHeader(gniFile)
    # Imports
    templateImports = """import("deqp.gni")
"""
    gniFile.write(templateImports)
    # Write the lists of data file dependencies
    templateDataFiles = """  "$root_gen_dir/vk_gl_cts_data/{dataFile}",
"""
    templateDataFileDeps = """
{dataDepName} = [
{files}]
"""
    for dataDirectory in dataDirectories:
        files = ""
        for dataFile in dataFiles:
            if dataDirectory + os.sep in dataFile:
                files += templateDataFiles.format(
                    dataFile=fixDestinationDirectory(pathReplacements, dataFile))
        dataDepName = "angle_deqp_" + convertPathToVarName(dataDirectory)
        fileDeps = templateDataFileDeps.format(dataDepName=dataDepName, files=files)
        gniFile.write(fileDeps)

    templateCopyTarget = """  "{deqpSupportDirectory}:vk_gl_cts_data_{relDir}",
"""
    templateCopyTargets = """
angle_deqp_data_copy_targets = [
{targets}]
"""
    targets = ""
    for relativeDirectory in relativeDirectories:
        targets += templateCopyTarget.format(
            deqpSupportDirectory=deqpSupportDirectory,
            relDir=convertPathToVarName(relativeDirectory))
    gniFile.write(templateCopyTargets.format(targets=targets))


if __name__ == '__main__':
    sys.exit(main())

kc3-lang/angle/scripts/gen_vk_gl_cts_build.py

Commit

kc3-lang/angle /scripts/gen_vk_gl_cts_build.py