Commit a064920e91bb85a023c4f1d682056c5549c15d7a
2019-06-24T23:27:47
Merge pull request #5140 from libgit2/ethomson/flaky_ci Re-run flaky tests
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
diff --git a/ci/test.ps1 b/ci/test.ps1
index 68b53e2..0c9e795 100644
--- a/ci/test.ps1
+++ b/ci/test.ps1
@@ -29,8 +29,32 @@ function run_test {
$TestCommand = (ctest -N -V -R "^$TestName$") -join "`n" -replace "(?ms).*\n^[0-9]*: Test command: ","" -replace "\n.*",""
$TestCommand += " -r${BuildDir}\results_${TestName}.xml"
- Invoke-Expression $TestCommand
- if ($LastExitCode -ne 0) { $global:Success = $false }
+ if ($Env:GITTEST_FLAKY_RETRY -gt 0) {
+ $AttemptsRemain = $Env:GITTEST_FLAKY_RETRY
+ } else {
+ $AttemptsRemain = 1
+ }
+
+ $Failed = 0
+ while ($AttemptsRemain -ne 0) {
+ if ($Failed -eq 1) {
+ Write-Host ""
+ Write-Host "Re-running flaky $TestName tests..."
+ Write-Host ""
+ }
+
+ Invoke-Expression $TestCommand
+ if ($LastExitCode -eq 0) {
+ $Failed = 0
+ break
+ } else {
+ $Failed = 1
+ }
+
+ $AttemptsRemain = $AttemptsRemain - 1
+ }
+
+ if ($Failed -eq 1) { $global:Success = $false }
}
Write-Host "##############################################################################"
@@ -79,7 +103,9 @@ if (-not $Env:SKIP_ONLINE_TESTS) {
Write-Host "## Running (online) tests"
Write-Host "##############################################################################"
+ $Env:GITTEST_FLAKY_RETRY=5
run_test online
+ $Env:GITTEST_FLAKY_RETRY=0
}
if (-not $Env:SKIP_PROXY_TESTS) {
diff --git a/ci/test.sh b/ci/test.sh
index e3caa80..9e12f53 100755
--- a/ci/test.sh
+++ b/ci/test.sh
@@ -32,11 +32,6 @@ cleanup() {
echo "Done."
}
-failure() {
- echo "Test exited with code: $1"
- SUCCESS=0
-}
-
# Ask ctest what it would run if we were to invoke it directly. This lets
# us manage the test configuration in a single place (tests/CMakeLists.txt)
# instead of running clar here as well. But it allows us to wrap our test
@@ -60,7 +55,35 @@ run_test() {
RUNNER="$TEST_CMD"
fi
- eval $RUNNER || failure
+ if [[ "$GITTEST_FLAKY_RETRY" > 0 ]]; then
+ ATTEMPTS_REMAIN=$GITTEST_FLAKY_RETRY
+ else
+ ATTEMPTS_REMAIN=1
+ fi
+
+ FAILED=0
+ while [[ "$ATTEMPTS_REMAIN" > 0 ]]; do
+ if [ "$FAILED" -eq 1 ]; then
+ echo ""
+ echo "Re-running flaky ${1} tests..."
+ echo ""
+ fi
+
+ RETURN_CODE=0
+ eval $RUNNER || RETURN_CODE=$? && true
+
+ if [ "$RETURN_CODE" -eq 0 ]; then
+ break
+ fi
+
+ echo "Test exited with code: $RETURN_CODE"
+ ATTEMPTS_REMAIN="$(($ATTEMPTS_REMAIN-1))"
+ FAILED=1
+ done
+
+ if [ "$FAILED" -ne 0 ]; then
+ SUCCESS=0
+ fi
}
# Configure the test environment; run them early so that we're certain
@@ -166,7 +189,9 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then
echo "## Running (online) tests"
echo "##############################################################################"
+ export GITTEST_FLAKY_RETRY=5
run_test online
+ unset GITTEST_FLAKY_RETRY
fi
if [ -z "$SKIP_GITDAEMON_TESTS" ]; then
@@ -238,7 +263,7 @@ fi
cleanup
-if [ "$SUCCESS" -ne "1" ]; then
+if [ "$SUCCESS" -ne 1 ]; then
echo "Some tests failed."
exit 1
fi