v/cmd/tools/repeat.v

372 lines
11 KiB
V
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

module main
import os
import flag
import time
import term
import math
import scripting
import v.util
struct CmdResult {
mut:
runs int
cmd string
icmd int
outputs []string
oms map[string][]int
summary map[string]Aints
timings []int
atiming Aints
}
struct Context {
mut:
count int
series int
warmup int
show_help bool
show_output bool
use_newline bool // use \n instead of \r, so the last line is not overwritten
fail_on_regress_percent int
fail_on_maxtime int // in ms
verbose bool
commands []string
results []CmdResult
cmd_template string // {T} will be substituted with the current command
cmd_params map[string][]string
cline string // a terminal clearing line
cgoback string
nmins int // number of minimums to discard
nmaxs int // number of maximums to discard
}
[unsafe]
fn (mut result CmdResult) free() {
unsafe {
result.cmd.free()
result.outputs.free()
result.oms.free()
result.summary.free()
result.timings.free()
result.atiming.free()
}
}
[unsafe]
fn (mut context Context) free() {
unsafe {
context.commands.free()
context.results.free()
context.cmd_template.free()
context.cmd_params.free()
context.cline.free()
context.cgoback.free()
}
}
struct Aints {
values []int
mut:
imin int
imax int
average f64
stddev f64
nmins int // number of discarded fastest results
nmaxs int // number of discarded slowest results
}
[unsafe]
fn (mut a Aints) free() {
unsafe { a.values.free() }
}
fn new_aints(ovals []int, extreme_mins int, extreme_maxs int) Aints {
mut res := Aints{
values: ovals // remember the original values
nmins: extreme_mins
nmaxs: extreme_maxs
}
mut sum := i64(0)
mut imin := math.max_i32
mut imax := -math.max_i32
// discard the extremes:
mut vals := []int{}
for x in ovals {
vals << x
}
vals.sort()
if vals.len > extreme_mins + extreme_maxs {
vals = vals[extreme_mins..vals.len - extreme_maxs].clone()
} else {
vals = []
}
// statistical processing of the remaining values:
for i in vals {
sum += i
if i < imin {
imin = i
}
if i > imax {
imax = i
}
}
res.imin = imin
res.imax = imax
if vals.len > 0 {
res.average = sum / f64(vals.len)
}
//
mut devsum := f64(0.0)
for i in vals {
x := f64(i) - res.average
devsum += (x * x)
}
res.stddev = math.sqrt(devsum / f64(vals.len))
// eprintln('\novals: $ovals\n vals: $vals\n vals.len: $vals.len | res.imin: $res.imin | res.imax: $res.imax | res.average: $res.average | res.stddev: $res.stddev')
return res
}
fn (a Aints) str() string {
return util.bold('${a.average:6.2f}') +
'ms ± σ: ${a.stddev:4.1f}ms, min: ${a.imin:4}ms, max: ${a.imax:4}ms, runs:${a.values.len:3}, nmins:${a.nmins:2}, nmaxs:${a.nmaxs:2}'
}
const (
max_fail_percent = 100 * 1000
max_time = 60 * 1000 // ms
performance_regression_label = 'Performance regression detected, failing since '
)
fn main() {
mut context := Context{}
context.parse_options()
context.run()
context.show_diff_summary()
}
fn (mut context Context) parse_options() {
mut fp := flag.new_flag_parser(os.args)
fp.application(os.file_name(os.executable()))
fp.version('0.0.1')
fp.description('Repeat command(s) and collect statistics. NB: you have to quote each command, if it contains spaces.')
fp.arguments_description('CMD1 CMD2 ...')
fp.skip_executable()
fp.limit_free_args_to_at_least(1)
context.count = fp.int('count', `c`, 10, 'Repetition count.')
context.series = fp.int('series', `s`, 2, 'Series count. `-s 2 -c 4 a b` => aaaabbbbaaaabbbb, while `-s 3 -c 2 a b` => aabbaabbaabb.')
context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs. These are done *only at the start*, and are ignored.')
context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
context.use_newline = fp.bool('newline', `n`, false, 'Use \\n, do not overwrite the last line. Produces more output, but easier to diagnose.')
context.show_output = fp.bool('output', `O`, false, 'Show command stdout/stderr in the progress indicator for each command. NB: slower, for verbose commands.')
context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
context.fail_on_maxtime = fp.int('max_time', `m`, max_time, 'Fail with exit code 2, when first cmd takes above M milliseconds (regression).')
context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with exit code 3, when first cmd is X% slower than the rest (regression).')
context.cmd_template = fp.string('template', `t`, '{T}', 'Command template. {T} will be substituted with the current command.')
cmd_params := fp.string_multi('parameter', `p`, 'A parameter substitution list. `{p}=val1,val2,val2` means that {p} in the template, will be substituted with each of val1, val2, val3.')
context.nmins = fp.int('nmins', `i`, 0, 'Ignore the BOTTOM X results (minimum execution time). Makes the results more robust to performance flukes.')
context.nmaxs = fp.int('nmaxs', `a`, 1, 'Ignore the TOP X results (maximum execution time). Makes the results more robust to performance flukes.')
for p in cmd_params {
parts := p.split(':')
if parts.len > 1 {
context.cmd_params[parts[0]] = parts[1].split(',')
}
}
if context.show_help {
println(fp.usage())
exit(0)
}
if context.verbose {
scripting.set_verbose(true)
}
commands := fp.finalize() or {
eprintln('Error: $err')
exit(1)
}
context.commands = context.expand_all_commands(commands)
context.results = []CmdResult{len: context.commands.len, cap: 20, init: CmdResult{
outputs: []string{cap: 500}
timings: []int{cap: 500}
}}
if context.use_newline {
context.cline = '\n'
context.cgoback = '\n'
} else {
context.cline = '\r' + term.h_divider('')
context.cgoback = '\r'
}
}
fn (mut context Context) clear_line() {
print(context.cline)
}
fn (mut context Context) expand_all_commands(commands []string) []string {
mut all_commands := []string{}
for cmd in commands {
maincmd := context.cmd_template.replace('{T}', cmd)
mut substituted_commands := []string{}
substituted_commands << maincmd
for paramk, paramlist in context.cmd_params {
for paramv in paramlist {
mut new_substituted_commands := []string{}
for cscmd in substituted_commands {
scmd := cscmd.replace(paramk, paramv)
new_substituted_commands << scmd
}
for sc in new_substituted_commands {
substituted_commands << sc
}
}
}
for sc in substituted_commands {
all_commands << sc
}
}
mut unique := map[string]int{}
for x in all_commands {
if x.contains('{') && x.contains('}') {
continue
}
unique[x] = 1
}
return unique.keys()
}
fn (mut context Context) run() {
mut run_warmups := 0
for si in 1 .. context.series + 1 {
for icmd, cmd in context.commands {
mut runs := 0
mut duration := 0
mut sum := 0
mut oldres := ''
println('Series: ${si:4}/${context.series:-4}, command: $cmd')
if context.warmup > 0 && run_warmups < context.commands.len {
for i in 1 .. context.warmup + 1 {
print('${context.cgoback}warming up run: ${i:4}/${context.warmup:-4} for ${cmd:-50s} took ${duration:6} ms ...')
mut sw := time.new_stopwatch({})
res := os.execute(cmd)
if res.exit_code != 0 {
continue
}
duration = int(sw.elapsed().milliseconds())
}
run_warmups++
}
context.clear_line()
for i in 1 .. (context.count + 1) {
avg := f64(sum) / f64(i)
print('${context.cgoback}Average: ${avg:9.3f}ms | run: ${i:4}/${context.count:-4} | took ${duration:6} ms')
if context.show_output {
print(' | result: ${oldres:s}')
}
mut sw := time.new_stopwatch({})
res := scripting.exec(cmd) or { continue }
duration = int(sw.elapsed().milliseconds())
if res.exit_code != 0 {
eprintln('${i:10} non 0 exit code for cmd: $cmd')
continue
}
trimed_output := res.output.trim_right('\r\n')
trimed_normalized := trimed_output.replace('\r\n', '\n')
lines := trimed_normalized.split('\n')
for line in lines {
context.results[icmd].outputs << line
}
context.results[icmd].timings << duration
sum += duration
runs++
oldres = res.output.replace('\n', ' ')
}
context.results[icmd].cmd = cmd
context.results[icmd].icmd = icmd
context.results[icmd].runs += runs
context.results[icmd].atiming = new_aints(context.results[icmd].timings, context.nmins,
context.nmaxs)
context.clear_line()
print(context.cgoback)
mut m := map[string][]int{}
ioutputs := context.results[icmd].outputs
for o in ioutputs {
x := o.split(':')
if x.len > 1 {
k := x[0]
v := x[1].trim_left(' ').int()
m[k] << v
}
}
mut summary := map[string]Aints{}
for k, v in m {
// show a temporary summary for the current series/cmd cycle
s := new_aints(v, context.nmins, context.nmaxs)
println(' $k: $s')
summary[k] = s
}
// merge current raw results to the previous ones
old_oms := context.results[icmd].oms.move()
mut new_oms := map[string][]int{}
for k, v in m {
if old_oms[k].len == 0 {
new_oms[k] = v
} else {
new_oms[k] << old_oms[k]
new_oms[k] << v
}
}
context.results[icmd].oms = new_oms.move()
// println('')
}
}
// create full summaries, taking account of all runs
for icmd in 0 .. context.results.len {
mut new_full_summary := map[string]Aints{}
for k, v in context.results[icmd].oms {
new_full_summary[k] = new_aints(v, context.nmins, context.nmaxs)
}
context.results[icmd].summary = new_full_summary.move()
}
}
fn (mut context Context) show_diff_summary() {
context.results.sort_with_compare(fn (a &CmdResult, b &CmdResult) int {
if a.atiming.average < b.atiming.average {
return -1
}
if a.atiming.average > b.atiming.average {
return 1
}
return 0
})
println('Summary (commands are ordered by ascending mean time), after $context.series series of $context.count repetitions:')
base := context.results[0].atiming.average
mut first_cmd_percentage := f64(100.0)
mut first_marker := ''
for i, r in context.results {
first_marker = ' '
cpercent := (r.atiming.average / base) * 100 - 100
if r.icmd == 0 {
first_marker = util.bold('>')
first_cmd_percentage = cpercent
}
println(' $first_marker${(i + 1):3} | ${cpercent:5.1f}% slower | ${r.cmd:-57s} | $r.atiming')
}
$if debugcontext ? {
println('context: $context')
}
if int(base) > context.fail_on_maxtime {
print(performance_regression_label)
println('average time: ${base:6.1f} ms > $context.fail_on_maxtime ms threshold.')
exit(2)
}
if context.fail_on_regress_percent == max_fail_percent || context.results.len < 2 {
return
}
fail_threshold_max := f64(context.fail_on_regress_percent)
if first_cmd_percentage > fail_threshold_max {
print(performance_regression_label)
println('${first_cmd_percentage:5.1f}% > ${fail_threshold_max:5.1f}% threshold.')
exit(3)
}
}