Commit c7b4ce55e065b1a4e50564f99b1832c0f3f38a18
2019-06-23T20:22:29
ci: add flaky test re-execution on Windows Our online tests are occasionally flaky since they hit real network endpoints. Re-run them up to 5 times if they fail, to allow us to avoid having to fail the whole build.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
diff --git a/ci/test.ps1 b/ci/test.ps1
index 68b53e2..0c9e795 100644
--- a/ci/test.ps1
+++ b/ci/test.ps1
@@ -29,8 +29,32 @@ function run_test {
$TestCommand = (ctest -N -V -R "^$TestName$") -join "`n" -replace "(?ms).*\n^[0-9]*: Test command: ","" -replace "\n.*",""
$TestCommand += " -r${BuildDir}\results_${TestName}.xml"
- Invoke-Expression $TestCommand
- if ($LastExitCode -ne 0) { $global:Success = $false }
+ if ($Env:GITTEST_FLAKY_RETRY -gt 0) {
+ $AttemptsRemain = $Env:GITTEST_FLAKY_RETRY
+ } else {
+ $AttemptsRemain = 1
+ }
+
+ $Failed = 0
+ while ($AttemptsRemain -ne 0) {
+ if ($Failed -eq 1) {
+ Write-Host ""
+ Write-Host "Re-running flaky $TestName tests..."
+ Write-Host ""
+ }
+
+ Invoke-Expression $TestCommand
+ if ($LastExitCode -eq 0) {
+ $Failed = 0
+ break
+ } else {
+ $Failed = 1
+ }
+
+ $AttemptsRemain = $AttemptsRemain - 1
+ }
+
+ if ($Failed -eq 1) { $global:Success = $false }
}
Write-Host "##############################################################################"
@@ -79,7 +103,9 @@ if (-not $Env:SKIP_ONLINE_TESTS) {
Write-Host "## Running (online) tests"
Write-Host "##############################################################################"
+ $Env:GITTEST_FLAKY_RETRY=5
run_test online
+ $Env:GITTEST_FLAKY_RETRY=0
}
if (-not $Env:SKIP_PROXY_TESTS) {