kc3-lang/angle/src/tests/test_utils/runner/TestSuite_unittest.cpp

• Show log

  Commit

• Hash : 87ed2c9d
  Author :
  Date : 2022-08-11T11:19:17
  

  TestSuiteTest: Generate more unique result file names.
  
  Speculative fix for flaky runs of TestSuiteTest. Flakes appear to read
  the results of other tests suggesting rand is not unique enough to
  protect against tests generating the same results file.
  
  Bug: chromium:1352134
  Change-Id: Icadf64c557e23e6d87fd18aa60aa48373ecec9fd
  Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3827144
  Commit-Queue: Geoff Lang <geofflang@chromium.org>
  Reviewed-by: Jonah Ryan-Davis <jonahr@google.com>
  

Download

• src/tests/test_utils/runner/TestSuite_unittest.cpp

• //
  // Copyright 2019 The ANGLE Project Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  //
  // TestSuite_unittest.cpp: Unit tests for ANGLE's test harness.
  //
  
  #include <gtest/gtest.h>
  
  #include "../angle_test_instantiate.h"
  #include "TestSuite.h"
  #include "common/debug.h"
  #include "common/system_utils.h"
  #include "util/test_utils.h"
  #include "util/test_utils_unittest_helper.h"
  
  #include <rapidjson/document.h>
  
  using namespace angle;
  
  namespace js = rapidjson;
  
  // This file is included in both angle_unittests and test_utils_unittest_helper. This variable is
  // defined separately in each test target's main file.
  extern bool gVerbose;
  
  namespace
  {
  constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper";
  constexpr int kFlakyRetries            = 3;
  
  class TestSuiteTest : public testing::Test
  {
    protected:
      void TearDown() override
      {
          if (!mTempFileName.empty())
          {
              angle::DeleteSystemFile(mTempFileName.c_str());
          }
      }
  
      bool runTestSuite(const std::vector<std::string> &extraArgs,
                        TestResults *actualResults,
                        bool validateStderr)
      {
          std::string executablePath = GetExecutableDirectory();
          EXPECT_NE(executablePath, "");
          executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
  
          const Optional<std::string> tempDirName = GetTempDirectory();
          if (!tempDirName.valid())
          {
              return false;
          }
  
          Optional<std::string> tempFile = CreateTemporaryFileInDirectory(tempDirName.value());
          if (!tempFile.valid())
          {
              return false;
          }
  
          mTempFileName               = tempFile.value();
          std::string resultsFileName = "--results-file=" + mTempFileName;
  
          std::vector<const char *> args = {
              executablePath.c_str(), kRunTestSuite,      "--gtest_also_run_disabled_tests",
              "--bot-mode",           "--test-timeout=5", resultsFileName.c_str()};
  
          for (const std::string &arg : extraArgs)
          {
              args.push_back(arg.c_str());
          }
  
          if (gVerbose)
          {
              printf("Test arguments:\n");
              for (const char *arg : args)
              {
                  printf("%s ", arg);
              }
              printf("\n");
          }
  
          ProcessHandle process(args, ProcessOutputCapture::StdoutAndStderrSeparately);
          EXPECT_TRUE(process->started());
          EXPECT_TRUE(process->finish());
          EXPECT_TRUE(process->finished());
  
          if (validateStderr)
          {
              EXPECT_EQ(process->getStderr(), "");
          }
  
          if (gVerbose)
          {
              printf("stdout:\n%s\n", process->getStdout().c_str());
          }
  
          return GetTestResultsFromFile(mTempFileName.c_str(), actualResults);
      }
  
      std::string mTempFileName;
  };
  
  // Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
  // Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
  TEST_F(TestSuiteTest, RunMockTests)
  {
      std::vector<std::string> extraArgs = {"--gtest_filter=MockTestSuiteTest.DISABLED_*"};
  
      TestResults actual;
      ASSERT_TRUE(runTestSuite(extraArgs, &actual, true));
  
      std::map<TestIdentifier, TestResult> expectedResults = {
          {{"MockTestSuiteTest", "DISABLED_Pass"},
           {TestResultType::Pass, std::vector<double>({0.0})}},
          {{"MockTestSuiteTest", "DISABLED_Fail"},
           {TestResultType::Fail, std::vector<double>({0.0})}},
          {{"MockTestSuiteTest", "DISABLED_Skip"},
           {TestResultType::Skip, std::vector<double>({0.0})}},
          {{"MockTestSuiteTest", "DISABLED_Timeout"},
           {TestResultType::Timeout, std::vector<double>({0.0})}},
      };
  
      EXPECT_EQ(expectedResults, actual.results);
  }
  
  // Verifies the flaky retry feature works as expected.
  TEST_F(TestSuiteTest, RunFlakyTests)
  {
      std::vector<std::string> extraArgs = {"--gtest_filter=MockFlakyTestSuiteTest.DISABLED_Flaky",
                                            "--flaky-retries=" + std::to_string(kFlakyRetries)};
  
      TestResults actual;
      ASSERT_TRUE(runTestSuite(extraArgs, &actual, true));
  
      std::vector<double> times;
      for (int i = 0; i < kFlakyRetries; i++)
      {
          times.push_back(0.0);
      }
      std::map<TestIdentifier, TestResult> expectedResults = {
          {{"MockFlakyTestSuiteTest", "DISABLED_Flaky"},
           {TestResultType::Pass, times, kFlakyRetries - 1}}};
  
      EXPECT_EQ(expectedResults, actual.results);
  }
  
  // Verifies that crashes are handled even without the crash handler.
  TEST_F(TestSuiteTest, RunCrashingTests)
  {
      std::vector<std::string> extraArgs = {
          "--gtest_filter=MockTestSuiteTest.DISABLED_Pass:MockTestSuiteTest.DISABLED_Fail:"
          "MockTestSuiteTest.DISABLED_Skip:"
          "MockCrashTestSuiteTest.DISABLED_*",
          "--disable-crash-handler"};
  
      TestResults actual;
      ASSERT_TRUE(runTestSuite(extraArgs, &actual, false));
  
      std::map<TestIdentifier, TestResult> expectedResults = {
          {{"MockTestSuiteTest", "DISABLED_Pass"},
           {TestResultType::Pass, std::vector<double>({0.0})}},
          {{"MockTestSuiteTest", "DISABLED_Fail"},
           {TestResultType::Fail, std::vector<double>({0.0})}},
          {{"MockTestSuiteTest", "DISABLED_Skip"},
           {TestResultType::Skip, std::vector<double>({0.0})}},
          {{"MockCrashTestSuiteTest", "DISABLED_Crash"},
           {TestResultType::Crash, std::vector<double>({0.0})}},
          {{"MockCrashTestSuiteTest", "DISABLED_PassAfterCrash"},
           {TestResultType::Pass, std::vector<double>({0.0})}},
          {{"MockCrashTestSuiteTest", "DISABLED_SkipAfterCrash"},
           {TestResultType::Skip, std::vector<double>({0.0})}},
      };
  
      EXPECT_EQ(expectedResults, actual.results);
  }
  
  // Normal passing test.
  TEST(MockTestSuiteTest, DISABLED_Pass)
  {
      EXPECT_TRUE(true);
  }
  
  // Normal failing test.
  TEST(MockTestSuiteTest, DISABLED_Fail)
  {
      EXPECT_TRUE(false);
  }
  
  // Trigger a test timeout.
  TEST(MockTestSuiteTest, DISABLED_Timeout)
  {
      angle::Sleep(20000);
  }
  
  // Trigger a test skip.
  TEST(MockTestSuiteTest, DISABLED_Skip)
  {
      GTEST_SKIP() << "Test skipped.";
  }
  
  // Trigger a flaky test failure.
  TEST(MockFlakyTestSuiteTest, DISABLED_Flaky)
  {
      const Optional<std::string> tempDirName = GetTempDirectory();
      ASSERT_TRUE(tempDirName.valid());
  
      std::stringstream tempFNameStream;
      tempFNameStream << tempDirName.value() << GetPathSeparator() << "flaky_temp.txt";
      std::string tempFileName = tempFNameStream.str();
  
      int fails = 0;
      {
          FILE *fp = fopen(tempFileName.c_str(), "r");
          if (fp)
          {
              ASSERT_EQ(fscanf(fp, "%d", &fails), 1);
              fclose(fp);
          }
      }
  
      if (fails >= kFlakyRetries - 1)
      {
          angle::DeleteSystemFile(tempFileName.c_str());
      }
      else
      {
          EXPECT_TRUE(false);
  
          FILE *fp = fopen(tempFileName.c_str(), "w");
          ASSERT_NE(fp, nullptr);
  
          fprintf(fp, "%d", fails + 1);
          fclose(fp);
      }
  }
  
  // Trigger a test crash.
  TEST(MockCrashTestSuiteTest, DISABLED_Crash)
  {
      ANGLE_CRASH();
  }
  
  // This test runs after the crash test.
  TEST(MockCrashTestSuiteTest, DISABLED_PassAfterCrash)
  {
      EXPECT_TRUE(true);
  }
  
  // This test runs after the crash test.
  TEST(MockCrashTestSuiteTest, DISABLED_SkipAfterCrash)
  {
      GTEST_SKIP() << "Test skipped.";
  }
  }  // namespace
  

Download