From c53ebd89b108759fc293f0feaa0a7306810eda3b Mon Sep 17 00:00:00 2001
From: Delyan Angelov <delian66@gmail.com>
Date: Fri, 2 Oct 2020 13:28:05 +0300
Subject: [PATCH] repeat.v: implement a --series option

---
 .github/workflows/ci.yml |   4 +-
 cmd/tools/repeat.v       | 130 ++++++++++++++++++++-------------------
 2 files changed, 70 insertions(+), 64 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cd000b78bc..c184a29b6c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,9 +39,9 @@ jobs:
     - name: Build the repeat tool
       run: ./v cmd/tools/repeat.v
     - name: Repeat -o v.c cmd/v
-      run: cmd/tools/repeat --count 10 --warmup 3 --fail_percent 10 './v -show-timings -o v.c cmd/v' './vmaster/v -show-timings -o v.c cmd/v'
+      run: cmd/tools/repeat --series 2 --count 10 --warmup 3 --fail_percent 10 './v -show-timings -o v.c cmd/v' './vmaster/v -show-timings -o v.c cmd/v'
     - name: Repeat -o hw.c examples/hello_world.v
-      run: cmd/tools/repeat --count 10 --warmup 3 --fail_percent 10 './v -show-timings -o hw.c examples/hello_world.v' './vmaster/v -show-timings -o hw.c examples/hello_world.v'
+      run: cmd/tools/repeat --series 2 --count 10 --warmup 3 --fail_percent 10 './v -show-timings -o hw.c examples/hello_world.v' './vmaster/v -show-timings -o hw.c examples/hello_world.v'
 
   ubuntu-tcc:
     runs-on: ubuntu-18.04
diff --git a/cmd/tools/repeat.v b/cmd/tools/repeat.v
index 0db65cc4f8..6a2e27a716 100644
--- a/cmd/tools/repeat.v
+++ b/cmd/tools/repeat.v
@@ -22,6 +22,7 @@ mut:
 struct Context {
 mut:
 	count int
+	series int
 	warmup int
 	show_help bool
 	show_result bool
@@ -84,12 +85,13 @@ fn (mut context Context) parse_options() {
 	mut fp := flag.new_flag_parser(os.args)
 	fp.application(os.file_name(os.executable()))
 	fp.version('0.0.1')
-	fp.description('Repeat command(s) and collect statistics. NB: you have to quote each command.')
+	fp.description('Repeat command(s) and collect statistics. NB: you have to quote each command, if it contains spaces.')
 	fp.arguments_description('CMD1 CMD2 ...')
 	fp.skip_executable()
 	fp.limit_free_args_to_at_least(1)
-	context.count = fp.int('count', `c`, 10, 'Repetition count')
-	context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs')
+	context.count = fp.int('count', `c`, 10, 'Repetition count.')
+	context.series = fp.int('series', `s`, 2, 'Series count. `-s 2 -c 4 a b` => aaaabbbbaaaabbbb, while `-s 3 -c 2 a b` => aabbaabbaabb.')
+	context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs. These are done *only at the start*, and are ignored.')
 	context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
 	context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
 	context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with 1 exit code, when first cmd is X% slower than the rest (regression).')
@@ -113,68 +115,72 @@ fn (mut context Context) clear_line() {
 }
 
 fn (mut context Context) run() {
-	for icmd, cmd in context.commands {
-		mut runs := 0
-		mut duration := 0
-		mut sum := 0
-		mut oldres := ''
-		println('Command: $cmd')
-		if context.warmup > 0 {
-			for i in 1..context.warmup+1 {
-				print('\r warming up run: ${i:4}/${context.warmup:-4} for ${cmd:-50s} took ${duration:6} ms ...')
+	mut run_warmups := 0
+	for si in 1..context.series+1 {
+		for icmd, cmd in context.commands {
+			mut runs := 0
+			mut duration := 0
+			mut sum := 0
+			mut oldres := ''
+			println('Series: ${si:4}/${context.series:-4}, command: $cmd')
+			if context.warmup > 0 && run_warmups < context.commands.len {
+				for i in 1..context.warmup+1 {
+					print('\r warming up run: ${i:4}/${context.warmup:-4} for ${cmd:-50s} took ${duration:6} ms ...')
+					mut sw := time.new_stopwatch({})
+					os.exec(cmd) or { continue }
+					duration = int(sw.elapsed().milliseconds())
+				}
+				run_warmups++
+			}
+			context.clear_line()
+			for i in 1..(context.count+1) {
+				avg := f64(sum)/f64(i)
+				print('\rAverage: ${avg:9.3f}ms | run: ${i:4}/${context.count:-4} | took ${duration:6} ms')
+				if context.show_result {
+					print(' | result: ${oldres:-s}')
+				}
 				mut sw := time.new_stopwatch({})
-				os.exec(cmd) or { continue }
+				res := os.exec(cmd) or {
+					eprintln('${i:10} failed runnning cmd: $cmd')
+					continue
+				}
 				duration = int(sw.elapsed().milliseconds())
+				if res.exit_code != 0 {
+					eprintln('${i:10} non 0 exit code for cmd: $cmd')
+					continue
+				}
+				context.results[icmd].outputs << res.output.trim_right('\r\n').replace('\r\n', '\n').split("\n")
+				context.results[icmd].timings << duration
+				sum += duration
+				runs++
+				oldres = res.output.replace('\n', ' ')
 			}
+			context.results[icmd].cmd = cmd
+			context.results[icmd].icmd = icmd
+			context.results[icmd].runs = runs
+			context.results[icmd].atiming = new_aints(context.results[icmd].timings)
+			context.clear_line()
+			print('\r')
+			mut m := map[string][]int
+			for o in context.results[icmd].outputs {
+				x := o.split(':')
+				if x.len > 1 {
+					k := x[0]
+					v := x[1].trim_left(' ').int()
+					m[k] << v
+				}
+			}
+			context.results[icmd].oms = m
+			oms := context.results[icmd].oms
+			mut summary := map[string]Aints{}
+			for k,v in oms {
+				s := new_aints(v)
+				println('  $k: $s')
+				summary[k] = s
+			}
+			context.results[icmd].summary = summary
+			//println('')
 		}
-		context.clear_line()
-		for i in 1..(context.count+1) {
-			avg := f64(sum)/f64(i)
-			print('\rAverage: ${avg:9.3f}ms | run: ${i:4}/${context.count:-4} | took ${duration:6} ms')
-			if context.show_result {
-				print(' | result: ${oldres:-s}')
-			}
-			mut sw := time.new_stopwatch({})
-			res := os.exec(cmd) or {
-				eprintln('${i:10} failed runnning cmd: $cmd')
-				continue
-			}
-			duration = int(sw.elapsed().milliseconds())
-			if res.exit_code != 0 {
-				eprintln('${i:10} non 0 exit code for cmd: $cmd')
-				continue
-			}
-			context.results[icmd].outputs << res.output.trim_right('\r\n').replace('\r\n', '\n').split("\n")
-			context.results[icmd].timings << duration
-			sum += duration
-			runs++
-			oldres = res.output.replace('\n', ' ')
-		}
-		context.results[icmd].cmd = cmd
-		context.results[icmd].icmd = icmd
-		context.results[icmd].runs = runs
-		context.results[icmd].atiming = new_aints(context.results[icmd].timings)
-		context.clear_line()
-		print('\r')
-		mut m := map[string][]int
-		for o in context.results[icmd].outputs {
-			x := o.split(':')
-			if x.len > 1 {
-				k := x[0]
-				v := x[1].trim_left(' ').int()
-				m[k] << v
-			}
-		}
-		context.results[icmd].oms = m
-		oms := context.results[icmd].oms
-		mut summary := map[string]Aints{}
-		for k,v in oms {
-			s := new_aints(v)
-			println('  $k: $s')
-			summary[k] = s
-		}
-		context.results[icmd].summary = summary
-		//println('')
 	}
 }
 fn (mut context Context) show_diff_summary() {
@@ -187,7 +193,7 @@ fn (mut context Context) show_diff_summary() {
 		}
 		return 0
 	})
-	println('Summary (commands are ordered by ascending mean time), after $context.count repeats:')
+	println('Summary (commands are ordered by ascending mean time), after $context.series series of $context.count repetitions:')
 	base := context.results[0].atiming.average
 	mut first_cmd_percentage := f64(100.0)
 	for i, r in context.results {