repeat: implement -nmaxs and -nmins to eliminate flukes

.github/workflows/ci.yml

@@ -63,9 +63,9 @@ jobs:
     - name: Build the repeat tool
       run: ./v cmd/tools/repeat.v
     - name: Repeat -o hw.c examples/hello_world.v
-      run: cmd/tools/repeat --max_time 150 --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o hw.c examples/hello_world.v' './vmaster/v -show-timings -o hw.c examples/hello_world.v'
+      run: cmd/tools/repeat --max_time 150 --series 3 --count 10 --nmaxs 5 --warmup 3 --fail_percent 10 -t '{T} -show-timings -o hw.c examples/hello_world.v' ./v ./vmaster/v
     - name: Repeat -o v.c cmd/v
-      run: cmd/tools/repeat --max_time 750 --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o v.c cmd/v' './vmaster/v -show-timings -o v.c cmd/v'
+      run: cmd/tools/repeat --max_time 750 --series 3 --count 15 --nmaxs 5 --warmup 3 --fail_percent 10 -t '{T} -show-timings -o v.c cmd/v' ./v ./vmaster/v
 
   ubuntu-tcc:
     runs-on: ubuntu-18.04

cmd/tools/repeat.v

@@ -37,6 +37,8 @@ mut:
 	cmd_params              map[string][]string
 	cline                   string // a terminal clearing line
 	cgoback                 string
+	nmins                   int // number of minimums to discard
+	nmaxs                   int // number of maximums to discard
 }
 
 struct Aints {
@@ -46,15 +48,28 @@ mut:
 	imax    int
 	average f64
 	stddev  f64
+	nmins   int // number of discarded fastest results
+	nmaxs   int // number of discarded slowest results
 }
 
-fn new_aints(vals []int) Aints {
+fn new_aints(ovals []int, extreme_mins int, extreme_maxs int) Aints {
 	mut res := Aints{
-		values: vals
+		values: ovals // remember the original values
+		nmins: extreme_mins
+		nmaxs: extreme_maxs
 	}
 	mut sum := i64(0)
 	mut imin := math.max_i32
 	mut imax := -math.max_i32
+	// discard the extremes:
+	mut vals := ovals.clone()
+	vals.sort()
+	if vals.len > extreme_mins + extreme_maxs {
+		vals = vals[extreme_mins..vals.len - extreme_maxs]
+	} else {
+		vals = []
+	}
+	// statistical processing of the remaining values:
 	for i in vals {
 		sum += i
 		if i < imin {
@@ -76,16 +91,17 @@ fn new_aints(vals []int) Aints {
 		devsum += (x * x)
 	}
 	res.stddev = math.sqrt(devsum / f64(vals.len))
+	// eprintln('\novals: $ovals\n vals: $vals\n vals.len: $vals.len |  res.imin: $res.imin | res.imax: $res.imax | res.average: $res.average | res.stddev: $res.stddev')
 	return res
 }
 
 fn (a Aints) str() string {
-	return util.bold('${a.average:9.3f}') +
+	return util.bold('${a.average:6.2f}') +
-		'ms ± σ: ${a.stddev:-5.1f}ms, min … max: ${a.imin}ms … ${a.imax}ms'
+		'ms ± σ: ${a.stddev:4.1f}ms, min: ${a.imin:4}ms, max: ${a.imax:4}ms, runs:${a.values.len:3}, nmins:${a.nmins:2}, nmaxs:${a.nmaxs:2}'
 }
 
 const (
-	max_fail_percent             = 100000
+	max_fail_percent             = 100 * 1000
 	max_time                     = 60 * 1000 // ms
 	performance_regression_label = 'Performance regression detected, failing since '
 )
@@ -109,13 +125,15 @@ fn (mut context Context) parse_options() {
 	context.series = fp.int('series', `s`, 2, 'Series count. `-s 2 -c 4 a b` => aaaabbbbaaaabbbb, while `-s 3 -c 2 a b` => aabbaabbaabb.')
 	context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs. These are done *only at the start*, and are ignored.')
 	context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
-	context.use_newline = fp.bool('newline', `N`, false, 'Use \\n, do not overwrite the last line. Produces more output, but easier to diagnose.')
+	context.use_newline = fp.bool('newline', `n`, false, 'Use \\n, do not overwrite the last line. Produces more output, but easier to diagnose.')
 	context.show_output = fp.bool('output', `O`, false, 'Show command stdout/stderr in the progress indicator for each command. NB: slower, for verbose commands.')
 	context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
 	context.fail_on_maxtime = fp.int('max_time', `m`, max_time, 'Fail with exit code 2, when first cmd takes above M milliseconds (regression).')
 	context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with exit code 3, when first cmd is X% slower than the rest (regression).')
 	context.cmd_template = fp.string('template', `t`, '{T}', 'Command template. {T} will be substituted with the current command.')
 	cmd_params := fp.string_multi('parameter', `p`, 'A parameter substitution list. `{p}=val1,val2,val2` means that {p} in the template, will be substituted with each of val1, val2, val3.')
+	context.nmins = fp.int('nmins', `i`, 0, 'Ignore the BOTTOM X results (minimum execution time). Makes the results more robust to performance flukes.')
+	context.nmaxs = fp.int('nmaxs', `a`, 1, 'Ignore the TOP X results (maximum execution time). Makes the results more robust to performance flukes.')
 	for p in cmd_params {
 		parts := p.split(':')
 		if parts.len > 1 {
@@ -217,7 +235,8 @@ fn (mut context Context) run() {
 			context.results[icmd].cmd = cmd
 			context.results[icmd].icmd = icmd
 			context.results[icmd].runs += runs
-			context.results[icmd].atiming = new_aints(context.results[icmd].timings)
+			context.results[icmd].atiming = new_aints(context.results[icmd].timings, context.nmins,
+				context.nmaxs)
 			context.clear_line()
 			print(context.cgoback)
 			mut m := map[string][]int{}
@@ -232,7 +251,7 @@ fn (mut context Context) run() {
 			mut summary := map[string]Aints{}
 			for k, v in m {
 				// show a temporary summary for the current series/cmd cycle
-				s := new_aints(v)
+				s := new_aints(v, context.nmins, context.nmaxs)
 				println('  $k: $s')
 				summary[k] = s
 			}
@@ -255,7 +274,7 @@ fn (mut context Context) run() {
 	for icmd in 0 .. context.results.len {
 		mut new_full_summary := map[string]Aints{}
 		for k, v in context.results[icmd].oms {
-			new_full_summary[k] = new_aints(v)
+			new_full_summary[k] = new_aints(v, context.nmins, context.nmaxs)
 		}
 		context.results[icmd].summary = new_full_summary
 	}
@@ -280,7 +299,7 @@ fn (mut context Context) show_diff_summary() {
 		if r.icmd == 0 {
 			first_cmd_percentage = cpercent
 		}
-		println(' $first_marker${(i + 1):3} | ${cpercent:6.1f}% slower | ${r.cmd:-55s} | $r.atiming')
+		println(' $first_marker${(i + 1):3} | ${cpercent:5.1f}% slower | ${r.cmd:-57s} | $r.atiming')
 	}
 	$if debugcontext ? {
 		println('context: $context')