kc3-lang/angle/scripts/perf_test_runner.py

• Show log

  Commit

• Hash : 1079bb2e
  Author :
  Date : 2016-11-17T10:00:39
  

  Fix up perf test runner script.
  
  The missing check was a bit buggy, and the paths were not updated
  to the new 'gyp' folder instead of 'build'.
  
  BUG=angleproject:1569
  
  Change-Id: I10408252851e90b79e306c8aea175ebe6a931b61
  Reviewed-on: https://chromium-review.googlesource.com/412320
  Reviewed-by: Geoff Lang <geofflang@chromium.org>
  

Download

• scripts/perf_test_runner.py

• #!/usr/bin/python
  #
  # Copyright 2015 The ANGLE Project Authors. All rights reserved.
  # Use of this source code is governed by a BSD-style license that can be
  # found in the LICENSE file.
  #
  # perf_test_runner.py:
  #   Helper script for running and analyzing perftest results. Runs the
  #   tests in an infinite batch, printing out the mean and standard
  #   deviation of the population continuously.
  #
  
  import subprocess
  import sys
  import os
  import re
  
  base_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
  
  # You might have to re-order these to find the specific version you want.
  perftests_paths = [
      os.path.join('out', 'Release_x64'),
      os.path.join('out', 'Release'),
      os.path.join('gyp', 'Release_x64'),
      os.path.join('gyp', 'Release_Win32')
  ]
  metric = 'score'
  
  # TODO(jmadill): Linux binaries
  binary_name = 'angle_perftests.exe'
  
  scores = []
  
  # Danke to http://stackoverflow.com/a/27758326
  def mean(data):
      """Return the sample arithmetic mean of data."""
      n = len(data)
      if n < 1:
          raise ValueError('mean requires at least one data point')
      return float(sum(data))/float(n) # in Python 2 use sum(data)/float(n)
  
  def _ss(data):
      """Return sum of square deviations of sequence data."""
      c = mean(data)
      ss = sum((float(x)-c)**2 for x in data)
      return ss
  
  def pstdev(data):
      """Calculates the population standard deviation."""
      n = len(data)
      if n < 2:
          raise ValueError('variance requires at least two data points')
      ss = _ss(data)
      pvar = ss/n # the population variance
      return pvar**0.5
  
  def truncated_list(data, n):
      """Compute a truncated list, n is truncation size"""
      if len(data) < n * 2:
          raise ValueError('list not large enough to truncate')
      return sorted(data)[n:-n]
  
  def truncated_mean(data, n):
      return mean(truncated_list(data, n))
  
  def truncated_stddev(data, n):
      return pstdev(truncated_list(data, n))
  
  # Find most recent binary
  newest_binary = None
  newest_mtime = None
  
  for path in perftests_paths:
      binary_path = os.path.join(base_path, path, binary_name)
      if os.path.exists(binary_path):
          binary_mtime = os.path.getmtime(binary_path)
          if (newest_binary is None) or (binary_mtime > newest_mtime):
              newest_binary = binary_path
              newest_mtime = binary_mtime
  
  perftests_path = newest_binary
  
  if perftests_path == None or not os.path.exists(perftests_path):
      print("Cannot find Release angle_perftests.exe!")
      sys.exit(1)
  
  test_name = "DrawCallPerfBenchmark.Run/d3d11_null"
  
  if len(sys.argv) >= 2:
      test_name = sys.argv[1]
  
  print('Using test executable: ' + perftests_path)
  print('Test name: ' + test_name)
  
  # Infinite loop of running the tests.
  while True:
      output = subprocess.check_output([perftests_path, '--gtest_filter=' + test_name])
  
      start_index = output.find(metric + "=")
      if start_index == -1:
          print("Did not find the score of the specified test in output:")
          print(output)
          sys.exit(1)
  
      start_index += len(metric) + 2
  
      end_index = output[start_index:].find(" ")
      if end_index == -1:
          print("Error parsing output:")
          print(output)
          sys.exit(2)
  
      m = re.search('Running (\d+) tests', output)
      if m and int(m.group(1)) > 1:
          print("Found more than one test result in output:")
          print(output)
          sys.exit(3)
  
      end_index += start_index
  
      score = int(output[start_index:end_index])
      sys.stdout.write("score: " + str(score))
  
      scores.append(score)
      sys.stdout.write(", mean: " + str(mean(scores)))
  
      if (len(scores) > 1):
          sys.stdout.write(", stddev: " + str(pstdev(scores)))
  
      if (len(scores) > 7):
          trucation_n = len(scores) >> 3
          sys.stdout.write(", truncated mean: " + str(truncated_mean(scores, trucation_n)))
          sys.stdout.write(", stddev: " + str(truncated_stddev(scores, trucation_n)))
  
      print("")
  

Download