Commit 6d8a34ad338eea7e5fe6c72b6a6dce7d4076cf09
2019-06-23T19:57:05
ci: add flaky test re-execution on Unix Our online tests are occasionally flaky since they hit real network endpoints. Re-run them up to 5 times if they fail, to allow us to avoid having to fail the whole build.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
diff --git a/ci/test.sh b/ci/test.sh
index e3caa80..9e12f53 100755
--- a/ci/test.sh
+++ b/ci/test.sh
@@ -32,11 +32,6 @@ cleanup() {
echo "Done."
}
-failure() {
- echo "Test exited with code: $1"
- SUCCESS=0
-}
-
# Ask ctest what it would run if we were to invoke it directly. This lets
# us manage the test configuration in a single place (tests/CMakeLists.txt)
# instead of running clar here as well. But it allows us to wrap our test
@@ -60,7 +55,35 @@ run_test() {
RUNNER="$TEST_CMD"
fi
- eval $RUNNER || failure
+ if [[ "$GITTEST_FLAKY_RETRY" > 0 ]]; then
+ ATTEMPTS_REMAIN=$GITTEST_FLAKY_RETRY
+ else
+ ATTEMPTS_REMAIN=1
+ fi
+
+ FAILED=0
+ while [[ "$ATTEMPTS_REMAIN" > 0 ]]; do
+ if [ "$FAILED" -eq 1 ]; then
+ echo ""
+ echo "Re-running flaky ${1} tests..."
+ echo ""
+ fi
+
+ RETURN_CODE=0
+ eval $RUNNER || RETURN_CODE=$? && true
+
+ if [ "$RETURN_CODE" -eq 0 ]; then
+ break
+ fi
+
+ echo "Test exited with code: $RETURN_CODE"
+ ATTEMPTS_REMAIN="$(($ATTEMPTS_REMAIN-1))"
+ FAILED=1
+ done
+
+ if [ "$FAILED" -ne 0 ]; then
+ SUCCESS=0
+ fi
}
# Configure the test environment; run them early so that we're certain
@@ -166,7 +189,9 @@ if [ -z "$SKIP_ONLINE_TESTS" ]; then
echo "## Running (online) tests"
echo "##############################################################################"
+ export GITTEST_FLAKY_RETRY=5
run_test online
+ unset GITTEST_FLAKY_RETRY
fi
if [ -z "$SKIP_GITDAEMON_TESTS" ]; then
@@ -238,7 +263,7 @@ fi
cleanup
-if [ "$SUCCESS" -ne "1" ]; then
+if [ "$SUCCESS" -ne 1 ]; then
echo "Some tests failed."
exit 1
fi