TestSuite.cpp

Branch :
Show log
Commit
Author : Jamie Madill
Date : 2020-09-16 19:30:56
Hash : 58463573
Message : Test Runner: Fix dirty thread teardown. When the standalone tests shut down they would not wait for the watchdog thread to exit cleanly. Fix this by setting a flag that shuts down the watchdog and attempting to wait. This was detected by running angle_end2end_tests with TSAN. Bug: angleproject:3162 Bug: b/168744561 Change-Id: I765ce7a21c3be11703619470a81adb1882b009e4 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2415175 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
src/tests/test_utils/runner/TestSuite.cpp
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite:
//   Basic implementation of a test harness in ANGLE.

#include "TestSuite.h"

#include "common/debug.h"
#include "common/platform.h"
#include "common/string_utils.h"
#include "common/system_utils.h"
#include "util/Timer.h"

#include <time.h>
#include <fstream>
#include <unordered_map>

#include <gtest/gtest.h>
#include <rapidjson/document.h>
#include <rapidjson/filewritestream.h>
#include <rapidjson/istreamwrapper.h>
#include <rapidjson/prettywriter.h>

// We directly call into a function to register the parameterized tests. This saves spinning up
// a subprocess with a new gtest filter.
#include <gtest/../../src/gtest-internal-inl.h>

namespace js = rapidjson;

namespace angle
{
namespace
{
constexpr char kTestTimeoutArg[]       = "--test-timeout=";
constexpr char kFilterFileArg[]        = "--filter-file=";
constexpr char kResultFileArg[]        = "--results-file=";
constexpr char kHistogramJsonFileArg[] = "--histogram-json-file=";
#if defined(NDEBUG)
constexpr int kDefaultTestTimeout = 20;
#else
constexpr int kDefaultTestTimeout  = 60;
#endif
#if defined(NDEBUG)
constexpr int kDefaultBatchTimeout = 240;
#else
constexpr int kDefaultBatchTimeout = 600;
#endif
constexpr int kDefaultBatchSize = 1000;

const char *ParseFlagValue(const char *flag, const char *argument)
{
    if (strstr(argument, flag) == argument)
    {
        return argument + strlen(flag);
    }

    return nullptr;
}

bool ParseIntArg(const char *flag, const char *argument, int *valueOut)
{
    const char *value = ParseFlagValue(flag, argument);
    if (!value)
    {
        return false;
    }

    char *end            = nullptr;
    const long longValue = strtol(value, &end, 10);

    if (*end != '\0')
    {
        printf("Error parsing integer flag value.\n");
        exit(1);
    }

    if (longValue == LONG_MAX || longValue == LONG_MIN || static_cast<int>(longValue) != longValue)
    {
        printf("Overflow when parsing integer flag value.\n");
        exit(1);
    }

    *valueOut = static_cast<int>(longValue);
    return true;
}

bool ParseFlag(const char *expected, const char *actual, bool *flagOut)
{
    if (strcmp(expected, actual) == 0)
    {
        *flagOut = true;
        return true;
    }
    return false;
}

bool ParseStringArg(const char *flag, const char *argument, std::string *valueOut)
{
    const char *value = ParseFlagValue(flag, argument);
    if (!value)
    {
        return false;
    }

    *valueOut = value;
    return true;
}

void DeleteArg(int *argc, char **argv, int argIndex)
{
    // Shift the remainder of the argv list left by one.  Note that argv has (*argc + 1) elements,
    // the last one always being NULL.  The following loop moves the trailing NULL element as well.
    for (int index = argIndex; index < *argc; ++index)
    {
        argv[index] = argv[index + 1];
    }
    (*argc)--;
}

void AddArg(int *argc, char **argv, const char *arg)
{
    // This unsafe const_cast is necessary to work around gtest limitations.
    argv[*argc]     = const_cast<char *>(arg);
    argv[*argc + 1] = nullptr;
    (*argc)++;
}

const char *ResultTypeToString(TestResultType type)
{
    switch (type)
    {
        case TestResultType::Crash:
            return "CRASH";
        case TestResultType::Fail:
            return "FAIL";
        case TestResultType::Pass:
            return "PASS";
        case TestResultType::Skip:
            return "SKIP";
        case TestResultType::Timeout:
            return "TIMEOUT";
        case TestResultType::Unknown:
            return "UNKNOWN";
    }
}

TestResultType GetResultTypeFromString(const std::string &str)
{
    if (str == "CRASH")
        return TestResultType::Crash;
    if (str == "FAIL")
        return TestResultType::Fail;
    if (str == "PASS")
        return TestResultType::Pass;
    if (str == "SKIP")
        return TestResultType::Skip;
    if (str == "TIMEOUT")
        return TestResultType::Timeout;
    return TestResultType::Unknown;
}

js::Value ResultTypeToJSString(TestResultType type, js::Document::AllocatorType *allocator)
{
    js::Value jsName;
    jsName.SetString(ResultTypeToString(type), *allocator);
    return jsName;
}

bool WriteJsonFile(const std::string &outputFile, js::Document *doc)
{
    FILE *fp = fopen(outputFile.c_str(), "w");
    if (!fp)
    {
        return false;
    }

    constexpr size_t kBufferSize = 0xFFFF;
    std::vector<char> writeBuffer(kBufferSize);
    js::FileWriteStream os(fp, writeBuffer.data(), kBufferSize);
    js::PrettyWriter<js::FileWriteStream> writer(os);
    if (!doc->Accept(writer))
    {
        fclose(fp);
        return false;
    }
    fclose(fp);
    return true;
}

// Writes out a TestResults to the Chromium JSON Test Results format.
// https://chromium.googlesource.com/chromium/src.git/+/master/docs/testing/json_test_results_format.md
void WriteResultsFile(bool interrupted,
                      const TestResults &testResults,
                      const std::string &outputFile,
                      const char *testSuiteName)
{
    time_t ltime;
    time(&ltime);
    struct tm *timeinfo = gmtime(&ltime);
    ltime               = mktime(timeinfo);

    uint64_t secondsSinceEpoch = static_cast<uint64_t>(ltime);

    js::Document doc;
    doc.SetObject();

    js::Document::AllocatorType &allocator = doc.GetAllocator();

    doc.AddMember("interrupted", interrupted, allocator);
    doc.AddMember("path_delimiter", ".", allocator);
    doc.AddMember("version", 3, allocator);
    doc.AddMember("seconds_since_epoch", secondsSinceEpoch, allocator);

    js::Value tests;
    tests.SetObject();

    std::map<TestResultType, uint32_t> counts;

    for (const auto &resultIter : testResults.results)
    {
        const TestIdentifier &id = resultIter.first;
        const TestResult &result = resultIter.second;

        js::Value jsResult;
        jsResult.SetObject();

        counts[result.type]++;

        jsResult.AddMember("expected", "PASS", allocator);
        jsResult.AddMember("actual", ResultTypeToJSString(result.type, &allocator), allocator);

        if (result.type != TestResultType::Pass)
        {
            jsResult.AddMember("is_unexpected", true, allocator);
        }

        js::Value times;
        times.SetArray();
        times.PushBack(result.elapsedTimeSeconds, allocator);

        jsResult.AddMember("times", times, allocator);

        char testName[500];
        id.sprintfName(testName);
        js::Value jsName;
        jsName.SetString(testName, allocator);

        tests.AddMember(jsName, jsResult, allocator);
    }

    js::Value numFailuresByType;
    numFailuresByType.SetObject();

    for (const auto &countIter : counts)
    {
        TestResultType type = countIter.first;
        uint32_t count      = countIter.second;

        js::Value jsCount(count);
        numFailuresByType.AddMember(ResultTypeToJSString(type, &allocator), jsCount, allocator);
    }

    doc.AddMember("num_failures_by_type", numFailuresByType, allocator);

    doc.AddMember("tests", tests, allocator);

    printf("Writing test results to %s\n", outputFile.c_str());

    if (!WriteJsonFile(outputFile, &doc))
    {
        printf("Error writing test results file.\n");
    }
}

void WriteHistogramJson(const TestResults &testResults,
                        const std::string &outputFile,
                        const char *testSuiteName)
{
    js::Document doc;
    doc.SetArray();

    // TODO: http://anglebug.com/4769 - Implement histogram output.

    printf("Writing histogram json to %s\n", outputFile.c_str());

    if (!WriteJsonFile(outputFile, &doc))
    {
        printf("Error writing histogram json file.\n");
    }
}

void WriteOutputFiles(bool interrupted,
                      const TestResults &testResults,
                      const std::string &resultsFile,
                      const std::string &histogramJsonOutputFile,
                      const char *testSuiteName)
{
    if (!resultsFile.empty())
    {
        WriteResultsFile(interrupted, testResults, resultsFile, testSuiteName);
    }

    if (!histogramJsonOutputFile.empty())
    {
        WriteHistogramJson(testResults, histogramJsonOutputFile, testSuiteName);
    }
}

void UpdateCurrentTestResult(const testing::TestResult &resultIn, TestResults *resultsOut)
{
    TestResult &resultOut = resultsOut->results[resultsOut->currentTest];

    // Note: Crashes and Timeouts are detected by the crash handler and a watchdog thread.
    if (resultIn.Skipped())
    {
        resultOut.type = TestResultType::Skip;
    }
    else if (resultIn.Failed())
    {
        resultOut.type = TestResultType::Fail;
    }
    else
    {
        resultOut.type = TestResultType::Pass;
    }

    resultOut.elapsedTimeSeconds = resultsOut->currentTestTimer.getElapsedTime();
}

TestIdentifier GetTestIdentifier(const testing::TestInfo &testInfo)
{
    return {testInfo.test_suite_name(), testInfo.name()};
}

class TestEventListener : public testing::EmptyTestEventListener
{
  public:
    // Note: TestResults is owned by the TestSuite. It should outlive TestEventListener.
    TestEventListener(const std::string &resultsFile,
                      const std::string &histogramJsonFile,
                      const char *testSuiteName,
                      TestResults *testResults)
        : mResultsFile(resultsFile),
          mHistogramJsonFile(histogramJsonFile),
          mTestSuiteName(testSuiteName),
          mTestResults(testResults)
    {}

    void OnTestStart(const testing::TestInfo &testInfo) override
    {
        std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
        mTestResults->currentTest = GetTestIdentifier(testInfo);
        mTestResults->currentTestTimer.start();
    }

    void OnTestEnd(const testing::TestInfo &testInfo) override
    {
        std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
        mTestResults->currentTestTimer.stop();
        const testing::TestResult &resultIn = *testInfo.result();
        UpdateCurrentTestResult(resultIn, mTestResults);
        mTestResults->currentTest = TestIdentifier();
    }

    void OnTestProgramEnd(const testing::UnitTest &testProgramInfo) override
    {
        std::lock_guard<std::mutex> guard(mTestResults->currentTestMutex);
        mTestResults->allDone = true;
        WriteOutputFiles(false, *mTestResults, mResultsFile, mHistogramJsonFile, mTestSuiteName);
    }

  private:
    std::string mResultsFile;
    std::string mHistogramJsonFile;
    const char *mTestSuiteName;
    TestResults *mTestResults;
};

bool IsTestDisabled(const testing::TestInfo &testInfo)
{
    return ::strstr(testInfo.name(), "DISABLED_") == testInfo.name();
}

using TestIdentifierFilter = std::function<bool(const TestIdentifier &id)>;

std::vector<TestIdentifier> FilterTests(std::map<TestIdentifier, FileLine> *fileLinesOut,
                                        TestIdentifierFilter filter,
                                        bool alsoRunDisabledTests)
{
    std::vector<TestIdentifier> tests;

    const testing::UnitTest &testProgramInfo = *testing::UnitTest::GetInstance();
    for (int suiteIndex = 0; suiteIndex < testProgramInfo.total_test_suite_count(); ++suiteIndex)
    {
        const testing::TestSuite &testSuite = *testProgramInfo.GetTestSuite(suiteIndex);
        for (int testIndex = 0; testIndex < testSuite.total_test_count(); ++testIndex)
        {
            const testing::TestInfo &testInfo = *testSuite.GetTestInfo(testIndex);
            TestIdentifier id                 = GetTestIdentifier(testInfo);
            if (filter(id) && (!IsTestDisabled(testInfo) || alsoRunDisabledTests))
            {
                tests.emplace_back(id);

                if (fileLinesOut)
                {
                    (*fileLinesOut)[id] = {testInfo.file(), testInfo.line()};
                }
            }
        }
    }

    return tests;
}

std::vector<TestIdentifier> GetFilteredTests(std::map<TestIdentifier, FileLine> *fileLinesOut,
                                             bool alsoRunDisabledTests)
{
    TestIdentifierFilter gtestIDFilter = [](const TestIdentifier &id) {
        return testing::internal::UnitTestOptions::FilterMatchesTest(id.testSuiteName, id.testName);
    };

    return FilterTests(fileLinesOut, gtestIDFilter, alsoRunDisabledTests);
}

std::vector<TestIdentifier> GetShardTests(const std::vector<TestIdentifier> &allTests,
                                          int shardIndex,
                                          int shardCount,
                                          std::map<TestIdentifier, FileLine> *fileLinesOut,
                                          bool alsoRunDisabledTests)
{
    std::vector<TestIdentifier> shardTests;

    for (int testIndex = shardIndex; testIndex < static_cast<int>(allTests.size());
         testIndex += shardCount)
    {
        shardTests.emplace_back(allTests[testIndex]);
    }

    return shardTests;
}

std::string GetTestFilter(const std::vector<TestIdentifier> &tests)
{
    std::stringstream filterStream;

    filterStream << "--gtest_filter=";

    for (size_t testIndex = 0; testIndex < tests.size(); ++testIndex)
    {
        if (testIndex != 0)
        {
            filterStream << ":";
        }

        filterStream << tests[testIndex];
    }

    return filterStream.str();
}

std::string ParseTestSuiteName(const char *executable)
{
    const char *baseNameStart = strrchr(executable, GetPathSeparator());
    if (!baseNameStart)
    {
        baseNameStart = executable;
    }
    else
    {
        baseNameStart++;
    }

    const char *suffix = GetExecutableExtension();
    size_t suffixLen   = strlen(suffix);
    if (suffixLen == 0)
    {
        return baseNameStart;
    }

    if (!EndsWith(baseNameStart, suffix))
    {
        return baseNameStart;
    }

    return std::string(baseNameStart, baseNameStart + strlen(baseNameStart) - suffixLen);
}

bool GetTestResultsFromJSON(const js::Document &document, TestResults *resultsOut)
{
    if (!document.HasMember("tests") || !document["tests"].IsObject())
    {
        return false;
    }

    const js::Value::ConstObject &tests = document["tests"].GetObject();
    for (auto iter = tests.MemberBegin(); iter != tests.MemberEnd(); ++iter)
    {
        // Get test identifier.
        const js::Value &name = iter->name;
        if (!name.IsString())
        {
            return false;
        }

        TestIdentifier id;
        if (!TestIdentifier::ParseFromString(name.GetString(), &id))
        {
            return false;
        }

        // Get test result.
        const js::Value &value = iter->value;
        if (!value.IsObject())
        {
            return false;
        }

        const js::Value::ConstObject &obj = value.GetObject();
        if (!obj.HasMember("expected") || !obj.HasMember("actual"))
        {
            return false;
        }

        const js::Value &expected = obj["expected"];
        const js::Value &actual   = obj["actual"];

        if (!expected.IsString() || !actual.IsString())
        {
            return false;
        }

        const std::string expectedStr = expected.GetString();
        const std::string actualStr   = actual.GetString();

        if (expectedStr != "PASS")
        {
            return false;
        }

        TestResultType resultType = GetResultTypeFromString(actualStr);
        if (resultType == TestResultType::Unknown)
        {
            return false;
        }

        double elapsedTimeSeconds = 0.0;
        if (obj.HasMember("times"))
        {
            const js::Value &times = obj["times"];
            if (!times.IsArray())
            {
                return false;
            }

            const js::Value::ConstArray &timesArray = times.GetArray();
            if (timesArray.Size() != 1 || !timesArray[0].IsDouble())
            {
                return false;
            }

            elapsedTimeSeconds = timesArray[0].GetDouble();
        }

        TestResult &result        = resultsOut->results[id];
        result.elapsedTimeSeconds = elapsedTimeSeconds;
        result.type               = resultType;
    }

    return true;
}

bool MergeTestResults(const TestResults &input, TestResults *output)
{
    for (const auto &resultsIter : input.results)
    {
        const TestIdentifier &id      = resultsIter.first;
        const TestResult &inputResult = resultsIter.second;
        TestResult &outputResult      = output->results[id];

        // This should probably handle situations where a test is run more than once.
        if (inputResult.type != TestResultType::Skip)
        {
            if (outputResult.type != TestResultType::Skip)
            {
                printf("Warning: duplicate entry for %s.%s.\n", id.testSuiteName.c_str(),
                       id.testName.c_str());
                return false;
            }

            outputResult.elapsedTimeSeconds = inputResult.elapsedTimeSeconds;
            outputResult.type               = inputResult.type;
        }
    }

    return true;
}

void PrintTestOutputSnippet(const TestIdentifier &id,
                            const TestResult &result,
                            const std::string &fullOutput)
{
    std::stringstream nameStream;
    nameStream << id;
    std::string fullName = nameStream.str();

    size_t runPos = fullOutput.find(std::string("[ RUN      ] ") + fullName);
    if (runPos == std::string::npos)
    {
        printf("Cannot locate test output snippet.\n");
        return;
    }

    size_t endPos = fullOutput.find(std::string("[  FAILED  ] ") + fullName, runPos);
    // Only clip the snippet to the "OK" message if the test really
    // succeeded. It still might have e.g. crashed after printing it.
    if (endPos == std::string::npos && result.type == TestResultType::Pass)
    {
        endPos = fullOutput.find(std::string("[       OK ] ") + fullName, runPos);
    }
    if (endPos != std::string::npos)
    {
        size_t newline_pos = fullOutput.find("\n", endPos);
        if (newline_pos != std::string::npos)
            endPos = newline_pos + 1;
    }

    std::cout << "\n";
    if (endPos != std::string::npos)
    {
        std::cout << fullOutput.substr(runPos, endPos - runPos);
    }
    else
    {
        std::cout << fullOutput.substr(runPos);
    }
    std::cout << "\n";
}

std::string GetConfigNameFromTestIdentifier(const TestIdentifier &id)
{
    size_t slashPos = id.testName.find('/');
    if (slashPos == std::string::npos)
    {
        return "default";
    }

    size_t doubleUnderscorePos = id.testName.find("__");
    if (doubleUnderscorePos == std::string::npos)
    {
        return id.testName.substr(slashPos + 1);
    }
    else
    {
        return id.testName.substr(slashPos + 1, doubleUnderscorePos - slashPos - 1);
    }
}

TestQueue BatchTests(const std::vector<TestIdentifier> &tests, int batchSize)
{
    // First sort tests by configuration.
    std::unordered_map<std::string, std::vector<TestIdentifier>> testsSortedByConfig;
    for (const TestIdentifier &id : tests)
    {
        std::string config = GetConfigNameFromTestIdentifier(id);
        testsSortedByConfig[config].push_back(id);
    }

    // Then group into batches by 'batchSize'.
    TestQueue testQueue;
    for (const auto &configAndIds : testsSortedByConfig)
    {
        const std::vector<TestIdentifier> &configTests = configAndIds.second;
        std::vector<TestIdentifier> batchTests;
        for (const TestIdentifier &id : configTests)
        {
            if (batchTests.size() >= static_cast<size_t>(batchSize))
            {
                testQueue.emplace(std::move(batchTests));
                ASSERT(batchTests.empty());
            }
            batchTests.push_back(id);
        }

        if (!batchTests.empty())
        {
            testQueue.emplace(std::move(batchTests));
        }
    }

    return testQueue;
}
}  // namespace

TestIdentifier::TestIdentifier() = default;

TestIdentifier::TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn)
    : testSuiteName(suiteNameIn), testName(nameIn)
{}

TestIdentifier::TestIdentifier(const TestIdentifier &other) = default;

TestIdentifier::~TestIdentifier() = default;

TestIdentifier &TestIdentifier::operator=(const TestIdentifier &other) = default;

void TestIdentifier::sprintfName(char *outBuffer) const
{
    sprintf(outBuffer, "%s.%s", testSuiteName.c_str(), testName.c_str());
}

// static
bool TestIdentifier::ParseFromString(const std::string &str, TestIdentifier *idOut)
{
    size_t separator = str.find(".");
    if (separator == std::string::npos)
    {
        return false;
    }

    idOut->testSuiteName = str.substr(0, separator);
    idOut->testName      = str.substr(separator + 1, str.length() - separator - 1);
    return true;
}

TestResults::TestResults() = default;

TestResults::~TestResults() = default;

ProcessInfo::ProcessInfo() = default;

ProcessInfo &ProcessInfo::operator=(ProcessInfo &&rhs)
{
    process         = std::move(rhs.process);
    testsInBatch    = std::move(rhs.testsInBatch);
    resultsFileName = std::move(rhs.resultsFileName);
    filterFileName  = std::move(rhs.filterFileName);
    commandLine     = std::move(rhs.commandLine);
    return *this;
}

ProcessInfo::~ProcessInfo() = default;

ProcessInfo::ProcessInfo(ProcessInfo &&other)
{
    *this = std::move(other);
}

TestSuite::TestSuite(int *argc, char **argv)
    : mShardCount(-1),
      mShardIndex(-1),
      mBotMode(false),
      mDebugTestGroups(false),
      mBatchSize(kDefaultBatchSize),
      mCurrentResultCount(0),
      mTotalResultCount(0),
      mMaxProcesses(NumberOfProcessors()),
      mTestTimeout(kDefaultTestTimeout),
      mBatchTimeout(kDefaultBatchTimeout)
{
    Optional<int> filterArgIndex;
    bool alsoRunDisabledTests = false;

#if defined(ANGLE_PLATFORM_WINDOWS)
    testing::GTEST_FLAG(catch_exceptions) = false;
#endif

    // Note that the crash callback must be owned and not use global constructors.
    mCrashCallback = [this]() { onCrashOrTimeout(TestResultType::Crash); };
    InitCrashHandler(&mCrashCallback);

    if (*argc <= 0)
    {
        printf("Missing test arguments.\n");
        exit(1);
    }

    mTestExecutableName = argv[0];
    mTestSuiteName      = ParseTestSuiteName(mTestExecutableName.c_str());

    for (int argIndex = 1; argIndex < *argc;)
    {
        if (parseSingleArg(argv[argIndex]))
        {
            DeleteArg(argc, argv, argIndex);
            continue;
        }

        if (ParseFlagValue("--gtest_filter=", argv[argIndex]))
        {
            filterArgIndex = argIndex;
        }
        else
        {
            // Don't include disabled tests in test lists unless the user asks for them.
            if (strcmp("--gtest_also_run_disabled_tests", argv[argIndex]) == 0)
            {
                alsoRunDisabledTests = true;
            }

            mGoogleTestCommandLineArgs.push_back(argv[argIndex]);
        }
        ++argIndex;
    }

    if ((mShardIndex >= 0) != (mShardCount > 1))
    {
        printf("Shard index and shard count must be specified together.\n");
        exit(1);
    }

    if (!mFilterFile.empty())
    {
        if (filterArgIndex.valid())
        {
            printf("Cannot use gtest_filter in conjunction with a filter file.\n");
            exit(1);
        }

        uint32_t fileSize = 0;
        if (!GetFileSize(mFilterFile.c_str(), &fileSize))
        {
            printf("Error getting filter file size: %s\n", mFilterFile.c_str());
            exit(1);
        }

        std::vector<char> fileContents(fileSize + 1, 0);
        if (!ReadEntireFileToString(mFilterFile.c_str(), fileContents.data(), fileSize))
        {
            printf("Error loading filter file: %s\n", mFilterFile.c_str());
            exit(1);
        }
        mFilterString.assign(fileContents.data());

        if (mFilterString.substr(0, strlen("--gtest_filter=")) != std::string("--gtest_filter="))
        {
            printf("Filter file must start with \"--gtest_filter=\".\n");
            exit(1);
        }

        // Note that we only add a filter string if we previously deleted a shader filter file
        // argument. So we will have space for the new filter string in argv.
        AddArg(argc, argv, mFilterString.c_str());
    }

    // Call into gtest internals to force parameterized test name registration.
    testing::internal::UnitTestImpl *impl = testing::internal::GetUnitTestImpl();
    impl->RegisterParameterizedTests();

    // Initialize internal GoogleTest filter arguments so we can call "FilterMatchesTest".
    testing::internal::ParseGoogleTestFlagsOnly(argc, argv);

    std::vector<TestIdentifier> testSet = GetFilteredTests(&mTestFileLines, alsoRunDisabledTests);

    if (mShardCount > 0)
    {
        if (!angle::GetEnvironmentVar("GTEST_SHARD_INDEX").empty() ||
            !angle::GetEnvironmentVar("GTEST_TOTAL_SHARDS").empty())
        {
            printf(
                "Error: --shard-index and --shard-count are incompatible with GTEST_SHARD_INDEX "
                "and GTEST_TOTAL_SHARDS.\n");
            exit(1);
        }

        testSet =
            GetShardTests(testSet, mShardIndex, mShardCount, &mTestFileLines, alsoRunDisabledTests);

        if (!mBotMode)
        {
            mFilterString = GetTestFilter(testSet);

            if (filterArgIndex.valid())
            {
                argv[filterArgIndex.value()] = const_cast<char *>(mFilterString.c_str());
            }
            else
            {
                // Note that we only add a filter string if we previously deleted a shard
                // index/count argument. So we will have space for the new filter string in argv.
                AddArg(argc, argv, mFilterString.c_str());
            }

            // Force-re-initialize GoogleTest flags to load the shard filter.
            testing::internal::ParseGoogleTestFlagsOnly(argc, argv);
        }
    }

    if (mBotMode)
    {
        // Split up test batches.
        mTestQueue = BatchTests(testSet, mBatchSize);

        if (mDebugTestGroups)
        {
            std::cout << "Test Groups:\n";

            while (!mTestQueue.empty())
            {
                const std::vector<TestIdentifier> &tests = mTestQueue.front();
                std::cout << tests[0] << " (" << static_cast<int>(tests.size()) << ")\n";
                mTestQueue.pop();
            }

            exit(0);
        }
    }

    testing::InitGoogleTest(argc, argv);

    mTotalResultCount = testSet.size();

    if ((mBotMode || !mResultsDirectory.empty()) && mResultsFile.empty())
    {
        // Create a default output file in bot mode.
        mResultsFile = "output.json";
    }

    if (!mResultsDirectory.empty())
    {
        std::stringstream resultFileName;
        resultFileName << mResultsDirectory << GetPathSeparator() << mResultsFile;
        mResultsFile = resultFileName.str();
    }

    if (!mResultsFile.empty() || !mHistogramJsonFile.empty())
    {
        testing::TestEventListeners &listeners = testing::UnitTest::GetInstance()->listeners();
        listeners.Append(new TestEventListener(mResultsFile, mHistogramJsonFile,
                                               mTestSuiteName.c_str(), &mTestResults));

        std::vector<TestIdentifier> testList = GetFilteredTests(nullptr, alsoRunDisabledTests);

        for (const TestIdentifier &id : testList)
        {
            mTestResults.results[id].type = TestResultType::Skip;
        }
    }
}

TestSuite::~TestSuite()
{
    if (mWatchdogThread.joinable())
    {
        mWatchdogThread.detach();
    }
    TerminateCrashHandler();
}

bool TestSuite::parseSingleArg(const char *argument)
{
    // Note: Flags should be documented in README.md.
    return (ParseIntArg("--shard-count=", argument, &mShardCount) ||
            ParseIntArg("--shard-index=", argument, &mShardIndex) ||
            ParseIntArg("--batch-size=", argument, &mBatchSize) ||
            ParseIntArg("--max-processes=", argument, &mMaxProcesses) ||
            ParseIntArg(kTestTimeoutArg, argument, &mTestTimeout) ||
            ParseIntArg("--batch-timeout=", argument, &mBatchTimeout) ||
            ParseStringArg("--results-directory=", argument, &mResultsDirectory) ||
            ParseStringArg(kResultFileArg, argument, &mResultsFile) ||
            ParseStringArg("--isolated-script-test-output=", argument, &mResultsFile) ||
            ParseStringArg(kFilterFileArg, argument, &mFilterFile) ||
            ParseStringArg(kHistogramJsonFileArg, argument, &mHistogramJsonFile) ||
            ParseStringArg("--isolated-script-test-perf-output=", argument, &mHistogramJsonFile) ||
            ParseFlag("--bot-mode", argument, &mBotMode) ||
            ParseFlag("--debug-test-groups", argument, &mDebugTestGroups));
}

void TestSuite::onCrashOrTimeout(TestResultType crashOrTimeout)
{
    if (mTestResults.currentTest.valid())
    {
        TestResult &result        = mTestResults.results[mTestResults.currentTest];
        result.type               = crashOrTimeout;
        result.elapsedTimeSeconds = mTestResults.currentTestTimer.getElapsedTime();
    }

    if (mResultsFile.empty())
    {
        printf("No results file specified.\n");
        return;
    }

    WriteOutputFiles(true, mTestResults, mResultsFile, mHistogramJsonFile, mTestSuiteName.c_str());
}

bool TestSuite::launchChildTestProcess(const std::vector<TestIdentifier> &testsInBatch)
{
    constexpr uint32_t kMaxPath = 1000;

    // Create a temporary file to store the test list
    ProcessInfo processInfo;

    char filterBuffer[kMaxPath] = {};
    if (!CreateTemporaryFile(filterBuffer, kMaxPath))
    {
        std::cerr << "Error creating temporary file for test list.\n";
        return false;
    }
    processInfo.filterFileName.assign(filterBuffer);

    std::string filterString = GetTestFilter(testsInBatch);

    FILE *fp = fopen(processInfo.filterFileName.c_str(), "w");
    if (!fp)
    {
        std::cerr << "Error opening temporary file for test list.\n";
        return false;
    }
    fprintf(fp, "%s", filterString.c_str());
    fclose(fp);

    std::string filterFileArg = kFilterFileArg + processInfo.filterFileName;

    // Create a temporary file to store the test output.
    char resultsBuffer[kMaxPath] = {};
    if (!CreateTemporaryFile(resultsBuffer, kMaxPath))
    {
        std::cerr << "Error creating temporary file for test list.\n";
        return false;
    }
    processInfo.resultsFileName.assign(resultsBuffer);

    std::string resultsFileArg = kResultFileArg + processInfo.resultsFileName;

    // Construct commandline for child process.
    std::vector<const char *> args;

    args.push_back(mTestExecutableName.c_str());
    args.push_back(filterFileArg.c_str());
    args.push_back(resultsFileArg.c_str());

    for (const std::string &arg : mGoogleTestCommandLineArgs)
    {
        args.push_back(arg.c_str());
    }

    std::string timeoutStr;
    if (mTestTimeout != kDefaultTestTimeout)
    {
        std::stringstream timeoutStream;
        timeoutStream << kTestTimeoutArg << mTestTimeout;
        timeoutStr = timeoutStream.str();
        args.push_back(timeoutStr.c_str());
    }

    // Launch child process and wait for completion.
    processInfo.process = LaunchProcess(args, true, true);

    if (!processInfo.process->started())
    {
        std::cerr << "Error launching child process.\n";
        return false;
    }

    std::stringstream commandLineStr;
    for (const char *arg : args)
    {
        commandLineStr << arg << " ";
    }

    processInfo.commandLine  = commandLineStr.str();
    processInfo.testsInBatch = testsInBatch;
    mCurrentProcesses.emplace_back(std::move(processInfo));
    return true;
}

bool TestSuite::finishProcess(ProcessInfo *processInfo)
{
    // Get test results and merge into master list.
    TestResults batchResults;

    if (!GetTestResultsFromFile(processInfo->resultsFileName.c_str(), &batchResults))
    {
        std::cerr << "Error reading test results from child process.\n";
        return false;
    }

    if (!MergeTestResults(batchResults, &mTestResults))
    {
        std::cerr << "Error merging batch test results.\n";
        return false;
    }

    if (!batchResults.results.empty())
    {
        const TestIdentifier &id = batchResults.results.begin()->first;
        std::string config       = GetConfigNameFromTestIdentifier(id);
        printf("Completed batch with config: %s\n", config.c_str());
    }

    // Process results and print unexpected errors.
    for (const auto &resultIter : batchResults.results)
    {
        const TestIdentifier &id = resultIter.first;
        const TestResult &result = resultIter.second;

        // Skip results aren't procesed since they're added back to the test queue below.
        if (result.type == TestResultType::Skip)
        {
            continue;
        }

        mCurrentResultCount++;
        printf("[%d/%d] %s.%s", mCurrentResultCount, mTotalResultCount, id.testSuiteName.c_str(),
               id.testName.c_str());

        if (result.type == TestResultType::Pass)
        {
            printf(" (%g ms)\n", result.elapsedTimeSeconds * 1000.0);
        }
        else
        {
            printf(" (%s)\n", ResultTypeToString(result.type));

            const std::string &batchStdout = processInfo->process->getStdout();
            PrintTestOutputSnippet(id, result, batchStdout);
        }
    }

    // On unexpected exit, re-queue any unfinished tests.
    if (processInfo->process->getExitCode() != 0)
    {
        std::vector<TestIdentifier> unfinishedTests;

        for (const auto &resultIter : batchResults.results)
        {
            const TestIdentifier &id = resultIter.first;
            const TestResult &result = resultIter.second;

            if (result.type == TestResultType::Skip)
            {
                unfinishedTests.push_back(id);
            }
        }

        mTestQueue.emplace(std::move(unfinishedTests));
    }

    // Clean up any dirty temporary files.
    for (const std::string &tempFile : {processInfo->filterFileName, processInfo->resultsFileName})
    {
        // Note: we should be aware that this cleanup won't happen if the harness itself crashes.
        // If this situation comes up in the future we should add crash cleanup to the harness.
        if (!angle::DeleteFile(tempFile.c_str()))
        {
            std::cerr << "Warning: Error cleaning up temp file: " << tempFile << "\n";
        }
    }

    processInfo->process.reset();
    return true;
}

int TestSuite::run()
{
    // Run tests serially.
    if (!mBotMode)
    {
        startWatchdog();
        int retVal = RUN_ALL_TESTS();

        {
            std::lock_guard<std::mutex> guard(mTestResults.currentTestMutex);
            mTestResults.allDone = true;
        }

        for (int tries = 0; tries < 10; ++tries)
        {
            if (!mWatchdogThread.joinable())
                break;
            angle::Sleep(100);
        }
        return retVal;
    }

    Timer totalRunTime;
    totalRunTime.start();

    constexpr double kIdleMessageTimeout = 5.0;

    Timer messageTimer;
    messageTimer.start();

    while (!mTestQueue.empty() || !mCurrentProcesses.empty())
    {
        bool progress = false;

        // Spawn a process if needed and possible.
        if (static_cast<int>(mCurrentProcesses.size()) < mMaxProcesses && !mTestQueue.empty())
        {
            std::vector<TestIdentifier> testsInBatch = mTestQueue.front();
            mTestQueue.pop();

            if (!launchChildTestProcess(testsInBatch))
            {
                return 1;
            }

            progress = true;
        }

        // Check for process completion.
        for (auto processIter = mCurrentProcesses.begin(); processIter != mCurrentProcesses.end();)
        {
            ProcessInfo &processInfo = *processIter;
            if (processInfo.process->finished())
            {
                if (!finishProcess(&processInfo))
                {
                    return 1;
                }
                processIter = mCurrentProcesses.erase(processIter);
                progress    = true;
            }
            else if (processInfo.process->getElapsedTimeSeconds() > mBatchTimeout)
            {
                // Terminate the process and record timeouts for the batch.
                // Because we can't determine which sub-test caused a timeout, record the whole
                // batch as a timeout failure. Can be improved by using socket message passing.
                if (!processInfo.process->kill())
                {
                    return 1;
                }
                for (const TestIdentifier &testIdentifier : processInfo.testsInBatch)
                {
                    // Because the whole batch failed we can't know how long each test took.
                    mTestResults.results[testIdentifier].type = TestResultType::Timeout;
                }

                processIter = mCurrentProcesses.erase(processIter);
                progress    = true;
            }
            else
            {
                processIter++;
            }
        }

        if (!progress && messageTimer.getElapsedTime() > kIdleMessageTimeout)
        {
            for (const ProcessInfo &processInfo : mCurrentProcesses)
            {
                double processTime = processInfo.process->getElapsedTimeSeconds();
                if (processTime > kIdleMessageTimeout)
                {
                    printf("Running for %d seconds: %s\n", static_cast<int>(processTime),
                           processInfo.commandLine.c_str());
                }
            }

            messageTimer.start();
        }

        // Sleep briefly and continue.
        angle::Sleep(100);
    }

    // Dump combined results.
    WriteOutputFiles(true, mTestResults, mResultsFile, mHistogramJsonFile, mTestSuiteName.c_str());

    totalRunTime.stop();
    printf("Tests completed in %lf seconds\n", totalRunTime.getElapsedTime());

    return printFailuresAndReturnCount() == 0;
}

int TestSuite::printFailuresAndReturnCount() const
{
    std::vector<std::string> failures;

    for (const auto &resultIter : mTestResults.results)
    {
        const TestIdentifier &id = resultIter.first;
        const TestResult &result = resultIter.second;

        if (result.type != TestResultType::Pass)
        {
            const FileLine &fileLine = mTestFileLines.find(id)->second;

            std::stringstream failureMessage;
            failureMessage << id << " (" << fileLine.file << ":" << fileLine.line << ") ("
                           << ResultTypeToString(result.type) << ")";
            failures.emplace_back(failureMessage.str());
        }
    }

    if (failures.empty())
        return 0;

    printf("%zu test%s failed:\n", failures.size(), failures.size() > 1 ? "s" : "");
    for (const std::string &failure : failures)
    {
        printf("    %s\n", failure.c_str());
    }

    return static_cast<int>(failures.size());
}

void TestSuite::startWatchdog()
{
    auto watchdogMain = [this]() {
        do
        {
            {
                std::lock_guard<std::mutex> guard(mTestResults.currentTestMutex);
                if (mTestResults.currentTestTimer.getElapsedTime() >
                    static_cast<double>(mTestTimeout))
                {
                    onCrashOrTimeout(TestResultType::Timeout);
                    exit(2);
                }

                if (mTestResults.allDone)
                    return;
            }

            angle::Sleep(1000);
        } while (true);
    };
    mWatchdogThread = std::thread(watchdogMain);
}

bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut)
{
    std::ifstream ifs(fileName);
    if (!ifs.is_open())
    {
        std::cerr << "Error opening " << fileName << "\n";
        return false;
    }

    js::IStreamWrapper ifsWrapper(ifs);
    js::Document document;
    document.ParseStream(ifsWrapper);

    if (document.HasParseError())
    {
        std::cerr << "Parse error reading JSON document: " << document.GetParseError() << "\n";
        return false;
    }

    if (!GetTestResultsFromJSON(document, resultsOut))
    {
        std::cerr << "Error getting test results from JSON.\n";
        return false;
    }

    return true;
}

const char *TestResultTypeToString(TestResultType type)
{
    switch (type)
    {
        case TestResultType::Crash:
            return "Crash";
        case TestResultType::Fail:
            return "Fail";
        case TestResultType::Skip:
            return "Skip";
        case TestResultType::Pass:
            return "Pass";
        case TestResultType::Timeout:
            return "Timeout";
        case TestResultType::Unknown:
            return "Unknown";
    }
}
}  // namespace angle
kc3-lang/angle/src/tests/test_utils/runner/TestSuite.cpp

Commit

kc3-lang/angle /src/tests/test_utils/runner/TestSuite.cpp
