kmx git

#! /usr/bin/env python3
#
# Copyright 2023 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
'''
compare_trace_screenshots.py

This script will cycle through screenshots from traces and compare them in useful ways.

It can run in multiple ways.

* `versus_native`

  This mode expects to be run in a directory full of two sets of screenshots

    angle_trace_tests --run-to-key-frame --screenshot-dir /tmp/screenshots
    angle_trace_tests --run-to-key-frame --screenshot-dir /tmp/screenshots --use-gl=native
    python3 compare_trace_screenshots.py versus_native --screenshot-dir /tmp/screenshots --trace-list-path ~/angle/src/tests/restricted_traces/

* `versus_upgrade`

  This mode expects to be pointed to two directories of identical images (same names and pixel contents)

    python3 compare_trace_screenshots.py versus_upgrade --before /my/trace/before --after /my/trace/after --out /my/trace/compare

Prerequisites
sudo apt-get install imagemagick
'''

import argparse
import json
import logging
import os
import subprocess
import sys

DEFAULT_LOG_LEVEL = 'info'

EXIT_SUCCESS = 0
EXIT_FAILURE = 1


def versus_native(args):

    # Get a list of all PNG files in the directory
    png_files = os.listdir(args.screenshot_dir)

    # Build a set of unique trace names
    traces = set()

    def get_traces_from_images():
        # Iterate through the PNG files
        for png_file in sorted(png_files):
            if png_file.startswith("angle_native") or png_file.startswith("angle_vulkan"):
                # Strip the prefix and the PNG extension from the file name
                trace_name = png_file.replace("angle_vulkan_",
                                              "").replace("swiftshader_",
                                                          "").replace("angle_native_",
                                                                      "").replace(".png", "")
                traces.add(trace_name)

    def get_traces_from_file(restricted_traces_path):
        with open(os.path.join(restricted_traces_path, "restricted_traces.json")) as f:
            trace_data = json.load(f)

        # Have to split the 'trace version' thing up
        trace_and_version = trace_data['traces']
        for i in trace_and_version:
            traces.add(i.split(' ',)[0])

    def get_trace_key_frame(restricted_traces_path, trace):
        with open(os.path.join(restricted_traces_path, trace, trace + ".json")) as f:
            single_trace_data = json.load(f)

        metadata = single_trace_data['TraceMetadata']
        keyframe = ""
        if 'KeyFrames' in metadata:
            keyframe = metadata['KeyFrames'][0]
        return keyframe

    if args.trace_list_path != None:
        get_traces_from_file(args.trace_list_path)
    else:
        get_traces_from_images()

    for trace in sorted(traces):
        if args.trace_list_path != None:
            keyframe = get_trace_key_frame(args.trace_list_path, trace)
            frame = ""
            if keyframe != "":
                frame = "_frame" + str(keyframe)

        native_file = "angle_native_" + trace + frame + ".png"
        native_file = os.path.join(args.screenshot_dir, native_file)
        if not os.path.isfile(native_file):
            native_file = "MISSING_EXT.png"

        vulkan_file = "angle_vulkan_" + trace + frame + ".png"
        vulkan_file = os.path.join(args.screenshot_dir, vulkan_file)
        if not os.path.isfile(vulkan_file):
            vulkan_file = "angle_vulkan_swiftshader_" + trace + frame + ".png"
            vulkan_file = os.path.join(args.screenshot_dir, vulkan_file)
            if not os.path.isfile(vulkan_file):
                vulkan_file = "MISSING_EXT.png"

        # Compare each of the images with different fuzz factors so we can see how each is doing
        # `compare -metric AE -fuzz ${FUZZ} ${VULKAN} ${NATIVE} ${TRACE}_fuzz${FUZZ}_diff.png`
        results = []
        for fuzz in {0, 1, 2, 5, 10, 20}:
            diff_file = trace + "_fuzz" + str(fuzz) + "%_TEST_diff.png"
            diff_file = os.path.join(args.screenshot_dir, diff_file)
            command = "compare -metric AE -fuzz " + str(
                fuzz) + "% " + vulkan_file + " " + native_file + " " + diff_file
            logging.debug("Running " + command)
            diff = subprocess.run(command, shell=True, capture_output=True)
            for line in diff.stderr.splitlines():
                if "unable to open image".encode('UTF-8') in line:
                    results.append("NA".encode('UTF-8'))
                else:
                    results.append(diff.stderr)
            logging.debug(" for " + trace + " " + str(fuzz) + "%")

        print(trace, os.path.basename(vulkan_file), os.path.basename(native_file),
              results[0].decode('UTF-8'), results[1].decode('UTF-8'), results[2].decode('UTF-8'),
              results[3].decode('UTF-8'), results[4].decode('UTF-8'), results[5].decode('UTF-8'))


def versus_upgrade(args):

    # Get a list of all the files in before
    before_files = sorted(os.listdir(args.before))

    # Get a list of all the files in after
    after_files = sorted(os.listdir(args.after))

    # If either list is missing files, this is a fail!
    if before_files != after_files:
        before_minus_after = list(sorted(set(before_files) - set(after_files)))
        after_minus_before = list(sorted(set(after_files) - set(before_files)))
        print("File lists don't match!")
        if before_minus_after is not []:
            print("Extra before files: %s" % before_minus_after)
        if after_minus_before is not []:
            print("Extra after files: %s" % after_minus_before)
        exit(1)

    # Walk through the before list and compare it with after
    for before_image, after_image in zip(sorted(before_files), sorted(after_files)):

        # Compare each of the images using root mean squared, no fuzz factor
        # `compare -metric RMSE ${BEFORE} ${AFTER} ${TRACE}_RMSE_diff.png;`

        results = []
        diff_file = args.outdir + "/" + before_image + "_TEST_diff.png"
        command = "compare -metric RMSE " + os.path.join(
            args.before, before_image) + " " + os.path.join(args.after,
                                                            after_image) + " " + diff_file
        diff = subprocess.run(command, shell=True, capture_output=True)
        for line in diff.stderr.splitlines():
            if "unable to open image".encode('UTF-8') in line:
                results.append("NA".encode('UTF-8'))
            else:
                # If the last element of the diff isn't zero, there was a pixel diff
                if line.split()[-1] != b'(0)':
                    print(before_image, diff.stderr.decode('UTF-8'))
                    print("Pixel diff detected!")
                    exit(1)
                else:
                    results.append(diff.stderr)

        print(before_image, results[0].decode('UTF-8'))

    print("Test completed successfully, no diffs detected")


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-l', '--log', help='Logging level.', default=DEFAULT_LOG_LEVEL)

    # Create commands for different modes of using this script
    subparsers = parser.add_subparsers(dest='command', required=True, help='Command to run.')

    # This mode will compare images of two runs, vulkan vs. native, and give you fuzzy comparison results
    versus_native_parser = subparsers.add_parser(
        'versus_native', help='Compares vulkan vs. native images.')
    versus_native_parser.add_argument(
        '--screenshot-dir', help='Directory containing two sets of screenshots', required=True)
    versus_native_parser.add_argument(
        '--trace-list-path', help='Path to dir containing restricted_traces.json')

    # This mode will compare before and after images when upgrading a trace
    versus_upgrade_parser = subparsers.add_parser(
        'versus_upgrade', help='Compare images before and after an upgrade')
    versus_upgrade_parser.add_argument(
        '--before', help='Full path to dir containing *before* screenshots', required=True)
    versus_upgrade_parser.add_argument(
        '--after', help='Full path to dir containing *after* screenshots', required=True)
    versus_upgrade_parser.add_argument('--outdir', help='Where to write output files', default='.')

    args = parser.parse_args()

    try:
        if args.command == 'versus_native':
            return versus_native(args)
        elif args.command == 'versus_upgrade':
            return versus_upgrade(args)
        else:
            logging.fatal('Unknown command: %s' % args.command)
            return EXIT_FAILURE
    except subprocess.CalledProcessError as e:
        logging.exception('There was an exception: %s', e.output.decode())
        return EXIT_FAILURE


if __name__ == '__main__':
    sys.exit(main())
kc3-lang/angle/src/tests/restricted_traces/compare_trace_screenshots.py

Commit

src/tests/restricted_traces/compare_trace_screenshots.py
