v/cmd/tools/repeat.v

372 lines
11 KiB
V
Raw Normal View History

module main
import os
import flag
import time
2020-10-01 22:25:29 +02:00
import term
import math
import scripting
2020-10-01 22:25:29 +02:00
import v.util
struct CmdResult {
mut:
runs int
cmd string
icmd int
outputs []string
oms map[string][]int
2020-10-01 22:25:29 +02:00
summary map[string]Aints
timings []int
2020-10-01 22:25:29 +02:00
atiming Aints
}
struct Context {
mut:
count int
series int
warmup int
show_help bool
show_output bool
use_newline bool // use \n instead of \r, so the last line is not overwritten
2020-10-02 09:57:58 +02:00
fail_on_regress_percent int
fail_on_maxtime int // in ms
verbose bool
commands []string
results []CmdResult
cmd_template string // {T} will be substituted with the current command
cmd_params map[string][]string
cline string // a terminal clearing line
cgoback string
nmins int // number of minimums to discard
nmaxs int // number of maximums to discard
}
[unsafe]
fn (mut result CmdResult) free() {
unsafe {
result.cmd.free()
result.outputs.free()
result.oms.free()
result.summary.free()
result.timings.free()
result.atiming.free()
}
}
[unsafe]
fn (mut context Context) free() {
unsafe {
context.commands.free()
context.results.free()
context.cmd_template.free()
context.cmd_params.free()
context.cline.free()
context.cgoback.free()
}
}
2020-10-01 22:25:29 +02:00
struct Aints {
values []int
2020-10-01 22:25:29 +02:00
mut:
imin int
imax int
2020-10-01 22:25:29 +02:00
average f64
stddev f64
nmins int // number of discarded fastest results
nmaxs int // number of discarded slowest results
2020-10-01 22:25:29 +02:00
}
[unsafe]
fn (mut a Aints) free() {
unsafe { a.values.free() }
}
fn new_aints(ovals []int, extreme_mins int, extreme_maxs int) Aints {
mut res := Aints{
values: ovals // remember the original values
nmins: extreme_mins
nmaxs: extreme_maxs
}
2020-10-01 22:25:29 +02:00
mut sum := i64(0)
mut imin := math.max_i32
2020-10-01 22:25:29 +02:00
mut imax := -math.max_i32
// discard the extremes:
mut vals := []int{}
for x in ovals {
vals << x
}
vals.sort()
if vals.len > extreme_mins + extreme_maxs {
vals = vals[extreme_mins..vals.len - extreme_maxs].clone()
} else {
vals = []
}
// statistical processing of the remaining values:
2020-10-01 22:25:29 +02:00
for i in vals {
sum += i
2020-10-01 22:25:29 +02:00
if i < imin {
imin = i
}
if i > imax {
imax = i
}
}
res.imin = imin
res.imax = imax
if vals.len > 0 {
res.average = sum / f64(vals.len)
}
//
mut devsum := f64(0.0)
for i in vals {
x := f64(i) - res.average
devsum += (x * x)
}
res.stddev = math.sqrt(devsum / f64(vals.len))
// eprintln('\novals: $ovals\n vals: $vals\n vals.len: $vals.len | res.imin: $res.imin | res.imax: $res.imax | res.average: $res.average | res.stddev: $res.stddev')
2020-10-01 22:25:29 +02:00
return res
}
fn (a Aints) str() string {
return util.bold('${a.average:6.2f}') +
'ms ± σ: ${a.stddev:4.1f}ms, min: ${a.imin:4}ms, max: ${a.imax:4}ms, runs:${a.values.len:3}, nmins:${a.nmins:2}, nmaxs:${a.nmaxs:2}'
}
2020-10-01 22:25:29 +02:00
2020-10-02 09:57:58 +02:00
const (
max_fail_percent = 100 * 1000
max_time = 60 * 1000 // ms
performance_regression_label = 'Performance regression detected, failing since '
2020-10-02 09:57:58 +02:00
)
fn main() {
mut context := Context{}
2020-10-01 19:46:45 +02:00
context.parse_options()
context.run()
context.show_diff_summary()
}
fn (mut context Context) parse_options() {
mut fp := flag.new_flag_parser(os.args)
fp.application(os.file_name(os.executable()))
fp.version('0.0.1')
2020-10-02 12:28:05 +02:00
fp.description('Repeat command(s) and collect statistics. NB: you have to quote each command, if it contains spaces.')
fp.arguments_description('CMD1 CMD2 ...')
fp.skip_executable()
fp.limit_free_args_to_at_least(1)
2020-10-02 12:28:05 +02:00
context.count = fp.int('count', `c`, 10, 'Repetition count.')
context.series = fp.int('series', `s`, 2, 'Series count. `-s 2 -c 4 a b` => aaaabbbbaaaabbbb, while `-s 3 -c 2 a b` => aabbaabbaabb.')
context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs. These are done *only at the start*, and are ignored.')
context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
context.use_newline = fp.bool('newline', `n`, false, 'Use \\n, do not overwrite the last line. Produces more output, but easier to diagnose.')
2020-10-02 17:10:25 +02:00
context.show_output = fp.bool('output', `O`, false, 'Show command stdout/stderr in the progress indicator for each command. NB: slower, for verbose commands.')
context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
context.fail_on_maxtime = fp.int('max_time', `m`, max_time, 'Fail with exit code 2, when first cmd takes above M milliseconds (regression).')
context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with exit code 3, when first cmd is X% slower than the rest (regression).')
context.cmd_template = fp.string('template', `t`, '{T}', 'Command template. {T} will be substituted with the current command.')
cmd_params := fp.string_multi('parameter', `p`, 'A parameter substitution list. `{p}=val1,val2,val2` means that {p} in the template, will be substituted with each of val1, val2, val3.')
context.nmins = fp.int('nmins', `i`, 0, 'Ignore the BOTTOM X results (minimum execution time). Makes the results more robust to performance flukes.')
context.nmaxs = fp.int('nmaxs', `a`, 1, 'Ignore the TOP X results (maximum execution time). Makes the results more robust to performance flukes.')
for p in cmd_params {
parts := p.split(':')
if parts.len > 1 {
context.cmd_params[parts[0]] = parts[1].split(',')
}
}
if context.show_help {
println(fp.usage())
exit(0)
}
if context.verbose {
scripting.set_verbose(true)
}
commands := fp.finalize() or {
eprintln('Error: $err')
exit(1)
}
context.commands = context.expand_all_commands(commands)
context.results = []CmdResult{len: context.commands.len, cap: 20, init: CmdResult{
outputs: []string{cap: 500}
timings: []int{cap: 500}
}}
if context.use_newline {
context.cline = '\n'
context.cgoback = '\n'
} else {
context.cline = '\r' + term.h_divider('')
context.cgoback = '\r'
}
2020-10-01 22:25:29 +02:00
}
fn (mut context Context) clear_line() {
print(context.cline)
2020-10-01 19:46:45 +02:00
}
fn (mut context Context) expand_all_commands(commands []string) []string {
mut all_commands := []string{}
for cmd in commands {
maincmd := context.cmd_template.replace('{T}', cmd)
mut substituted_commands := []string{}
substituted_commands << maincmd
for paramk, paramlist in context.cmd_params {
for paramv in paramlist {
mut new_substituted_commands := []string{}
for cscmd in substituted_commands {
scmd := cscmd.replace(paramk, paramv)
new_substituted_commands << scmd
}
for sc in new_substituted_commands {
substituted_commands << sc
}
}
}
for sc in substituted_commands {
all_commands << sc
}
}
mut unique := map[string]int{}
for x in all_commands {
if x.contains('{') && x.contains('}') {
continue
}
unique[x] = 1
}
return unique.keys()
}
2020-10-01 19:46:45 +02:00
fn (mut context Context) run() {
2020-10-02 12:28:05 +02:00
mut run_warmups := 0
for si in 1 .. context.series + 1 {
2020-10-02 12:28:05 +02:00
for icmd, cmd in context.commands {
mut runs := 0
mut duration := 0
mut sum := 0
mut oldres := ''
println('Series: ${si:4}/${context.series:-4}, command: $cmd')
if context.warmup > 0 && run_warmups < context.commands.len {
for i in 1 .. context.warmup + 1 {
print('${context.cgoback}warming up run: ${i:4}/${context.warmup:-4} for ${cmd:-50s} took ${duration:6} ms ...')
2020-10-02 12:28:05 +02:00
mut sw := time.new_stopwatch({})
res := os.execute(cmd)
if res.exit_code != 0 {
continue
}
2020-10-02 12:28:05 +02:00
duration = int(sw.elapsed().milliseconds())
}
run_warmups++
}
context.clear_line()
for i in 1 .. (context.count + 1) {
avg := f64(sum) / f64(i)
print('${context.cgoback}Average: ${avg:9.3f}ms | run: ${i:4}/${context.count:-4} | took ${duration:6} ms')
2020-10-02 17:10:25 +02:00
if context.show_output {
print(' | result: ${oldres:s}')
2020-10-02 12:28:05 +02:00
}
mut sw := time.new_stopwatch({})
res := scripting.exec(cmd) or { continue }
duration = int(sw.elapsed().milliseconds())
2020-10-02 12:28:05 +02:00
if res.exit_code != 0 {
eprintln('${i:10} non 0 exit code for cmd: $cmd')
continue
}
trimed_output := res.output.trim_right('\r\n')
trimed_normalized := trimed_output.replace('\r\n', '\n')
lines := trimed_normalized.split('\n')
for line in lines {
context.results[icmd].outputs << line
}
2020-10-02 12:28:05 +02:00
context.results[icmd].timings << duration
sum += duration
runs++
oldres = res.output.replace('\n', ' ')
}
2020-10-02 12:28:05 +02:00
context.results[icmd].cmd = cmd
context.results[icmd].icmd = icmd
context.results[icmd].runs += runs
context.results[icmd].atiming = new_aints(context.results[icmd].timings, context.nmins,
context.nmaxs)
2020-10-02 12:28:05 +02:00
context.clear_line()
print(context.cgoback)
mut m := map[string][]int{}
ioutputs := context.results[icmd].outputs
for o in ioutputs {
2020-10-02 12:28:05 +02:00
x := o.split(':')
if x.len > 1 {
k := x[0]
v := x[1].trim_left(' ').int()
m[k] << v
}
}
2020-10-02 12:28:05 +02:00
mut summary := map[string]Aints{}
for k, v in m {
// show a temporary summary for the current series/cmd cycle
s := new_aints(v, context.nmins, context.nmaxs)
2020-10-02 12:28:05 +02:00
println(' $k: $s')
summary[k] = s
}
// merge current raw results to the previous ones
old_oms := context.results[icmd].oms.move()
mut new_oms := map[string][]int{}
for k, v in m {
if old_oms[k].len == 0 {
new_oms[k] = v
} else {
new_oms[k] << old_oms[k]
new_oms[k] << v
}
}
context.results[icmd].oms = new_oms.move()
// println('')
2020-10-01 22:25:29 +02:00
}
}
// create full summaries, taking account of all runs
for icmd in 0 .. context.results.len {
mut new_full_summary := map[string]Aints{}
for k, v in context.results[icmd].oms {
new_full_summary[k] = new_aints(v, context.nmins, context.nmaxs)
}
context.results[icmd].summary = new_full_summary.move()
}
2020-10-01 19:46:45 +02:00
}
2020-10-01 19:46:45 +02:00
fn (mut context Context) show_diff_summary() {
context.results.sort_with_compare(fn (a &CmdResult, b &CmdResult) int {
2020-10-01 22:25:29 +02:00
if a.atiming.average < b.atiming.average {
return -1
}
if a.atiming.average > b.atiming.average {
return 1
}
return 0
})
2020-10-02 12:28:05 +02:00
println('Summary (commands are ordered by ascending mean time), after $context.series series of $context.count repetitions:')
2020-10-01 22:25:29 +02:00
base := context.results[0].atiming.average
2020-10-02 09:57:58 +02:00
mut first_cmd_percentage := f64(100.0)
mut first_marker := ''
2020-10-01 22:25:29 +02:00
for i, r in context.results {
first_marker = ' '
2020-10-02 09:57:58 +02:00
cpercent := (r.atiming.average / base) * 100 - 100
if r.icmd == 0 {
first_marker = util.bold('>')
2020-10-02 09:57:58 +02:00
first_cmd_percentage = cpercent
}
println(' $first_marker${(i + 1):3} | ${cpercent:5.1f}% slower | ${r.cmd:-57s} | $r.atiming')
2020-10-02 09:57:58 +02:00
}
$if debugcontext ? {
println('context: $context')
}
if int(base) > context.fail_on_maxtime {
print(performance_regression_label)
println('average time: ${base:6.1f} ms > $context.fail_on_maxtime ms threshold.')
exit(2)
}
2020-10-02 09:57:58 +02:00
if context.fail_on_regress_percent == max_fail_percent || context.results.len < 2 {
return
}
fail_threshold_max := f64(context.fail_on_regress_percent)
if first_cmd_percentage > fail_threshold_max {
print(performance_regression_label)
2020-10-02 09:57:58 +02:00
println('${first_cmd_percentage:5.1f}% > ${fail_threshold_max:5.1f}% threshold.')
exit(3)
2020-10-01 22:25:29 +02:00
}
}