tests: support for marking _test.v files with `// vtest flaky: true`, to reduce false positives from the CI

Delyan Angelov 2022-04-30 13:27:50 +03:00 committed by Jef Roosens
parent 1ad84bb2d1
commit 76c67952e6
Signed by: Jef Roosens
GPG Key ID: B75D4F293C7052DB
15 changed files with 30 additions and 2 deletions

View File

@ -19,6 +19,8 @@ pub const hide_oks = os.getenv('VTEST_HIDE_OK') == '1'
pub const fail_fast = os.getenv('VTEST_FAIL_FAST') == '1'
pub const fail_flaky = os.getenv('VTEST_FAIL_FLAKY') == '1'
pub const test_only = os.getenv('VTEST_ONLY').split_any(',')
pub const test_only_fn = os.getenv('VTEST_ONLY_FN').split_any(',')
@ -362,7 +364,7 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
details := get_test_details(file)
os.setenv('VTEST_RETRY_MAX', '$details.retry', true)
for retry := 1; retry <= details.retry; retry++ {
ts.append_message(.info, ' retrying $retry/$details.retry of $relative_file ...')
ts.append_message(.info, ' [stats] retrying $retry/$details.retry of $relative_file ; known flaky: $details.flaky ...')
os.setenv('VTEST_RETRY', '$retry', true)
status = os.system(cmd)
if status == 0 {
@ -372,6 +374,10 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
}
time.sleep(500 * time.millisecond)
}
if details.flaky && !testing.fail_flaky {
ts.append_message(.info, ' *FAILURE* of the known flaky test file $relative_file is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: $details.retry .')
goto test_passed_system
}
ts.failed = true
ts.benchmark.fail()
tls_bench.fail()
@ -399,7 +405,7 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
details := get_test_details(file)
os.setenv('VTEST_RETRY_MAX', '$details.retry', true)
for retry := 1; retry <= details.retry; retry++ {
ts.append_message(.info, ' retrying $retry/$details.retry of $relative_file ...')
ts.append_message(.info, ' retrying $retry/$details.retry of $relative_file ; known flaky: $details.flaky ...')
os.setenv('VTEST_RETRY', '$retry', true)
r = os.execute(cmd)
if r.exit_code == 0 {
@ -408,6 +414,10 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
}
}
}
if details.flaky && !testing.fail_flaky {
ts.append_message(.info, ' *FAILURE* of the known flaky test file $relative_file is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: $details.retry .')
goto test_passed_system
}
ts.failed = true
ts.benchmark.fail()
tls_bench.fail()
@ -562,6 +572,7 @@ pub fn setup_new_vtmp_folder() string {
pub struct TestDetails {
pub mut:
retry int
flaky bool // when flaky tests fail, the whole run is still considered successfull, unless VTEST_FAIL_FLAKY is 1
}
pub fn get_test_details(file string) TestDetails {
@ -571,6 +582,9 @@ pub fn get_test_details(file string) TestDetails {
if line.starts_with('// vtest retry:') {
res.retry = line.all_after(':').trim_space().int()
}
if line.starts_with('// vtest flaky:') {
res.flaky = line.all_after(':').trim_space().bool()
}
}
return res
}

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
import context

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 8
import net
import os

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
import os
import os.notify

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
import os
import time

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 15
// This test case runs concurrent 3 instances of `do_select` that

View File

@ -1,5 +1,6 @@
module sync
// vtest flaky: true
// vtest retry: 6
// ATTENTION! Do not use this file as an example!

View File

@ -1,6 +1,7 @@
import os
import time
// vtest flaky: true
// vtest retry: 4
/*

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
import os
import time

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
fn sum1(a int, b int) int {

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
fn f(x f64) f64 {

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
fn f(n int) ?f64 {

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 4
import sync

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 4
/*
* To verify the effect of "[keep_args_alive]", this attribute may be commented out.

View File

@ -1,3 +1,4 @@
// vtest flaky: true
// vtest retry: 3
import sync
import time