kc3-lang/angle/src/tests/test_utils/runner/TestSuite.h

• Show log

  Commit

• Hash : f65d6acc
  Author :
  Date : 2021-08-18T12:20:40
  

  Test Runner: Determine slow tests from test expectations.
  
  This updates the test harness to handle slow test based on the TIMEOUT
  test expectation. It removes the "registerSlowTests" API in favor of
  using the test expectation files.
  
  Also updates some of the timeout handling for various angle tests,
  including on SwiftShader for multithreading tests.
  
  Increases the timeout for tests in a batch to attempt to fix the
  problem of the first test being much slower.
  
  Bug: angleproject:6261
  Change-Id: I7427344da5a0c1ea26d8c2d6eb0e9be2557b56e6
  Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3104007
  Commit-Queue: Jamie Madill <jmadill@chromium.org>
  Reviewed-by: Geoff Lang <geofflang@chromium.org>
  Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
  

Download

• src/tests/test_utils/runner/TestSuite.h

• //
  // Copyright 2019 The ANGLE Project Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  //
  // TestSuite:
  //   Basic implementation of a test harness in ANGLE.
  
  #ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
  #define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
  
  #include <map>
  #include <memory>
  #include <mutex>
  #include <queue>
  #include <string>
  #include <thread>
  
  #include "HistogramWriter.h"
  #include "tests/test_expectations/GPUTestExpectationsParser.h"
  #include "util/test_utils.h"
  
  namespace angle
  {
  struct TestIdentifier
  {
      TestIdentifier();
      TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn);
      TestIdentifier(const TestIdentifier &other);
      ~TestIdentifier();
  
      TestIdentifier &operator=(const TestIdentifier &other);
  
      static bool ParseFromString(const std::string &str, TestIdentifier *idOut);
  
      bool valid() const { return !testName.empty(); }
      void sprintfName(char *outBuffer) const;
  
      std::string testSuiteName;
      std::string testName;
  };
  
  inline bool operator<(const TestIdentifier &a, const TestIdentifier &b)
  {
      return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName);
  }
  
  inline bool operator==(const TestIdentifier &a, const TestIdentifier &b)
  {
      return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName);
  }
  
  inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id)
  {
      return os << id.testSuiteName << "." << id.testName;
  }
  
  enum class TestResultType
  {
      Crash,
      Fail,
      NoResult,
      Pass,
      Skip,
      Timeout,
      Unknown,
  };
  
  const char *TestResultTypeToString(TestResultType type);
  
  struct TestResult
  {
      TestResultType type       = TestResultType::NoResult;
      double elapsedTimeSeconds = 0.0;
      uint32_t flakyFailures    = 0;
  };
  
  inline bool operator==(const TestResult &a, const TestResult &b)
  {
      return a.type == b.type;
  }
  
  inline std::ostream &operator<<(std::ostream &os, const TestResult &result)
  {
      return os << TestResultTypeToString(result.type);
  }
  
  struct TestResults
  {
      TestResults();
      ~TestResults();
  
      std::map<TestIdentifier, TestResult> results;
      std::mutex currentTestMutex;
      TestIdentifier currentTest;
      Timer currentTestTimer;
      double currentTestTimeout = 0.0;
      bool allDone              = false;
      std::string testArtifactsFakeTestName;
      std::vector<std::string> testArtifactPaths;
  };
  
  struct FileLine
  {
      const char *file;
      int line;
  };
  
  struct ProcessInfo : angle::NonCopyable
  {
      ProcessInfo();
      ~ProcessInfo();
      ProcessInfo(ProcessInfo &&other);
      ProcessInfo &operator=(ProcessInfo &&rhs);
  
      ProcessHandle process;
      std::vector<TestIdentifier> testsInBatch;
      std::string resultsFileName;
      std::string filterFileName;
      std::string commandLine;
      std::string filterString;
  };
  
  using TestQueue = std::queue<std::vector<TestIdentifier>>;
  
  class TestSuite
  {
    public:
      TestSuite(int *argc, char **argv);
      ~TestSuite();
  
      int run();
      void onCrashOrTimeout(TestResultType crashOrTimeout);
      void addHistogramSample(const std::string &measurement,
                              const std::string &story,
                              double value,
                              const std::string &units);
  
      static TestSuite *GetInstance() { return mInstance; }
  
      // Returns the path to the artifact in the output directory.
      std::string addTestArtifact(const std::string &artifactName);
  
      int getShardIndex() const { return mShardIndex; }
      int getBatchId() const { return mBatchId; }
  
      // Test expectation processing.
      bool loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config,
                                                  const std::string &fileName);
      bool loadAllTestExpectationsFromFile(const std::string &fileName);
      int32_t getTestExpectation(const std::string &testName);
      void maybeUpdateTestTimeout(uint32_t testExpectation);
      int32_t getTestExpectationWithConfigAndUpdateTimeout(const GPUTestConfig &config,
                                                           const std::string &testName);
      bool logAnyUnusedTestExpectations();
      void setTestExpectationsAllowMask(uint32_t mask)
      {
          mTestExpectationsParser.setTestExpectationsAllowMask(mask);
      }
  
    private:
      bool parseSingleArg(const char *argument);
      bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch);
      bool finishProcess(ProcessInfo *processInfo);
      int printFailuresAndReturnCount() const;
      void startWatchdog();
      void dumpTestExpectationsErrorMessages();
      int getSlowTestTimeout() const;
  
      static TestSuite *mInstance;
  
      std::string mTestExecutableName;
      std::string mTestSuiteName;
      TestQueue mTestQueue;
      std::string mFilterString;
      std::string mFilterFile;
      std::string mResultsDirectory;
      std::string mResultsFile;
      std::string mHistogramJsonFile;
      int mShardCount;
      int mShardIndex;
      angle::CrashCallback mCrashCallback;
      TestResults mTestResults;
      bool mBotMode;
      bool mDebugTestGroups;
      bool mGTestListTests;
      bool mListTests;
      bool mPrintTestStdout;
      bool mDisableCrashHandler;
      int mBatchSize;
      int mCurrentResultCount;
      int mTotalResultCount;
      int mMaxProcesses;
      int mTestTimeout;
      int mBatchTimeout;
      int mBatchId;
      int mFlakyRetries;
      int mMaxFailures;
      int mFailureCount;
      std::vector<std::string> mChildProcessArgs;
      std::map<TestIdentifier, FileLine> mTestFileLines;
      std::vector<ProcessInfo> mCurrentProcesses;
      std::thread mWatchdogThread;
      HistogramWriter mHistogramWriter;
      std::string mTestArtifactDirectory;
      GPUTestExpectationsParser mTestExpectationsParser;
  };
  
  bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
  }  // namespace angle
  
  #endif  // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
  

Download