From f43c4fd81cbf946644193fe2b52023f7eaf737e9 Mon Sep 17 00:00:00 2001
From: Delyan Angelov <delian66@gmail.com>
Date: Fri, 4 Oct 2019 16:11:29 +0300
Subject: [PATCH] tools/performance_compare: speed it up and make it more
 robust

* tools/performance_compare is now more robust. It uses the C source from the time of the v commit, instead of always the latest one. It also now clones https://github.com/vlang/vc just once per comparison, not for every build.

* Remove obsoleted tools/compare_v_performance_between_commits shell script.
---
 tools/compare_v_performance_between_commits | 98 ---------------------
 tools/performance_compare.v                 | 96 ++++++++++++++------
 2 files changed, 71 insertions(+), 123 deletions(-)
 delete mode 100755 tools/compare_v_performance_between_commits

diff --git a/tools/compare_v_performance_between_commits b/tools/compare_v_performance_between_commits
deleted file mode 100755
index e6c849f483..0000000000
--- a/tools/compare_v_performance_between_commits
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/bin/sh
-
-set -e 
-
-msg() { 
-    printf '%s\n' "$*"; 
-}
-
-if [ $# -ne 2 ]; then
-    msg "Usage: compare_v_to_c_performance COMMIT_BEFORE COMMIT_AFTER"
-    exit 1
-fi
-
-depend_on() { 
-    type "$1" >/dev/null 2>&1 || {
-        printf 'ERR: missing tool "%s"\n' "$1" >&2; exit 1;
-    } 
-}
-
-depend_on sh
-depend_on cp
-depend_on rm
-depend_on wc
-depend_on head
-depend_on cc
-depend_on strip
-depend_on git
-depend_on upx
-depend_on make
-depend_on hyperfine
-
-######################################################################
-## NB: cc should be a working, recent, sane C99 compiler 
-##     cc is used by the Makefile to bootstrap v (both gcc/clang work)
-##
-## If you are a C/V developer in a unix environment, you most probably 
-## already have the above installed, with the possible exception of:
-## https://github.com/sharkdp/hyperfine
-##
-## Installing them is out of scope of this tool.
-######################################################################
-
-COMMIT_B="$1"
-COMMIT_A="$2"
-
-CWD="$(pwd)"
-WORKDIR="/tmp"
-
-B="$WORKDIR/v_at_$COMMIT_B"
-A="$WORKDIR/v_at_$COMMIT_A"
-
-prepare_v() {
-    msg 
-    msg "Cloning current v source to $1 ..."
-    git clone --quiet "$CWD" "$1"
-    
-    cd "$1"
-    git checkout --quiet "$2"
-    
-    msg "Making v and vprod compilers in $1"
-    make > /dev/null
-    ./v -o v compiler 
-    ./v -prod -o vprod compiler
-    cp v     v_stripped 
-    cp vprod vprod_stripped 
-    strip *_stripped
-    cp v_stripped      v_stripped_upxed
-    cp vprod_stripped  vprod_stripped_upxed
-    upx -qqq --lzma v_stripped_upxed 
-    upx -qqq --lzma vprod_stripped_upxed
-    wc -c "$1/v" "$1/v_stripped" "$1/v_stripped_upxed" "$1/vprod" "$1/vprod_stripped" "$1/vprod_stripped_upxed" | head -n -1
-    VVERSION="$($1/v --version)"
-    GVERSION="$(git rev-parse --short  --verify HEAD)"
-    msg "V version is: $VVERSION , local source commit: $GVERSION"
-}
-
-compare_v_performance() {
-    CMD="$1"
-    msg "---------------------------------------------------------------------------------"
-    msg "Compare '$CMD'"
-    hyperfine --warmup=3 "cd '$B/' && $CMD "  "cd '$A/' && $CMD "
-    msg
-}
-
-##############################################################################
-# Cleanup artifacts from previous runs of this tool:
-cd "$WORKDIR"
-rm -rf "$A/" "$B/"
-##############################################################################
-
-msg "Comparing v compiler performance of commit $COMMIT_B (before) vs commit $COMMIT_A (after) ..."
-prepare_v "$B" "$COMMIT_B"
-prepare_v "$A" "$COMMIT_A"
-
-cd "$WORKDIR"
-compare_v_performance "./v     -o x.c compiler"
-compare_v_performance "./vprod -o x.c compiler"
-compare_v_performance "./vprod -o x   compiler"
diff --git a/tools/performance_compare.v b/tools/performance_compare.v
index f886565e32..ff61dc23e0 100644
--- a/tools/performance_compare.v
+++ b/tools/performance_compare.v
@@ -6,7 +6,7 @@ const (
 	tool_version = '0.0.3'
 	tool_description = '' +
 		'  Compares V executable size and performance,\n' +
-		'  between 2 commits from V\'s local git history.\n' + 
+		'  between 2 commits from V\'s local git history.\n' +
 		'  When only one commit is given, it is compared to master.'
 )
 
@@ -16,6 +16,7 @@ mut:
 	workdir       string // the working folder (typically /tmp), where the tool will write
 	a             string // the full path to the 'after' folder inside workdir
 	b             string // the full path to the 'before' folder inside workdir
+	vc            string // the full path to the vc folder inside workdir. It is used during bootstrapping v from the C source.
 	commit_before string // the git commit for the 'before' state
 	commit_after  string // the git commit for the 'after' state
 }
@@ -44,7 +45,7 @@ fn tool_must_exist(toolcmd string) {
 }
 
 fn used_tools_must_exist(tools []string) {
-	for t in tools { 
+	for t in tools {
 		tool_must_exist(t)
 	}
 }
@@ -54,16 +55,18 @@ fn (c Context) compare_versions() {
 	// Input is validated at this point...
 	//Cleanup artifacts from previous runs of this tool:
 	os.chdir( c.workdir )
-	run('rm -rf "$c.a" "$c.b" ')
-	
+	run('rm -rf "$c.a" "$c.b" "$c.vc" ')
+	// clone the VC source *just once per comparison*, and reuse it:
+	run('git clone --quiet https://github.com/vlang/vc	\'$c.vc\' ')
+
 	println('Comparing v compiler performance of commit $c.commit_before (before) vs commit $c.commit_after (after) ...')
 	c.prepare_v( c.b , c.commit_before )
 	c.prepare_v( c.a , c.commit_after  )
-	
+
 	os.chdir( c.workdir )
-	c.compare_v_performance( 'v     -o source.c compiler' )
+	c.compare_v_performance( 'v		-o source.c compiler' )
 	c.compare_v_performance( 'vprod -o source.c compiler' )
-	c.compare_v_performance( 'vprod -o binary   compiler' )
+	c.compare_v_performance( 'vprod -o binary	compiler' )
 }
 
 fn show_sizes_of_files(files []string) {
@@ -73,24 +76,66 @@ fn show_sizes_of_files(files []string) {
 	}
 }
 
+fn line_to_timestamp_and_commit(line string) (int, string) {
+	parts := line.split(' ')
+	return parts[0].int(), parts[1]
+}
+
+fn (c &Context) prepare_vc_source( cdir string, commit string ) {
+	os.chdir( cdir )
+	// Building a historic v with the latest vc is not always possible ...
+	// It is more likely, that the vc *at the time of the v commit*,
+	// or slightly before that time will be able to build the historic v:
+	vline := run('git rev-list -n1 --timestamp \'$commit\' ')
+	v_timestamp, _ := line_to_timestamp_and_commit( vline )
+	os.chdir( c.vc )
+	run('git checkout master')
+	vcbefore := run('git rev-list HEAD -n1 --timestamp --before=$v_timestamp ')
+	_, vccommit_before := line_to_timestamp_and_commit( vcbefore )
+	run('git checkout \'$vccommit_before\' ')
+	os.chdir( cdir )
+}
+
 fn (c &Context) prepare_v( cdir string, commit string ) {
+	mut cc := os.getenv('CC')
+	if cc == '' { cc = 'cc' }
+
+	mut command_for_building_v_from_c_source := '$cc -std=gnu11 -w -o cv	 $c.vc/v.c	   -lm'
+	if 'windows' == os.user_os() {
+		command_for_building_v_from_c_source  = '$cc -std=gnu11 -w -o cv.exe $c.vc/v_win.c'
+	}
+
 	println('')
+	// prepare c.vc first
+	os.chdir( c.vc )
+	run('git checkout master')
+
 	println('Cloning current v source to $cdir ...')
 	os.system('git clone --quiet \'$c.cwd\' \'$cdir\' ')
 	os.chdir( cdir )
 	os.system('git checkout --quiet \'$commit\' ')
-	
+
+	run('git clean -f')
+	c.prepare_vc_source( cdir, commit )
+
 	println('Making v and vprod compilers in $cdir')
-	run('make')
-	run('./v       -o v     compiler/ ')
-	run('./v -prod -o vprod compiler/ ')
-	run('cp v     v_stripped')
-	run('cp vprod vprod_stripped')
+	run(command_for_building_v_from_c_source)
+	run('./cv		-o v	 compiler/ ')
+	run('./cv -prod -o vprod compiler/ ')
+
+	run('cp cv		 cv_stripped')
+	run('cp v		 v_stripped')
+	run('cp vprod	 vprod_stripped')
 	run('strip *_stripped')
-	run('cp v_stripped      v_stripped_upxed')
-	run('cp vprod_stripped  vprod_stripped_upxed')
+
+	run('cp cv_stripped		 cv_stripped_upxed')
+	run('cp v_stripped		 v_stripped_upxed')
+	run('cp vprod_stripped	 vprod_stripped_upxed')
+	run('upx -qqq --lzma cv_stripped_upxed')
 	run('upx -qqq --lzma v_stripped_upxed')
 	run('upx -qqq --lzma vprod_stripped_upxed')
+
+	show_sizes_of_files(["$cdir/cv",    "$cdir/cv_stripped",     "$cdir/cv_stripped_upxed"])
 	show_sizes_of_files(["$cdir/v",     "$cdir/v_stripped",      "$cdir/v_stripped_upxed"])
 	show_sizes_of_files(["$cdir/vprod", "$cdir/vprod_stripped",  "$cdir/vprod_stripped_upxed"])
 	println("V version is: " + run("$cdir/v --version") + " , local source commit: " + run("git rev-parse --short  --verify HEAD") )
@@ -114,14 +159,14 @@ fn (c Context) normalized_workpath_for_commit( commit string ) string {
 fn validate_commit_exists( commit string ){
 	cmd := 'git cat-file -t ' + "'" + commit + "'"
 	if !command_exits_with_zero_status(cmd) {
-		eprintln("Commit: '" +  commit + "' does not exist in the current repository.")
+		eprintln("Commit: '" + commit + "' does not exist in the current repository.")
 		exit(3)
 	}
 }
 
 fn main(){
 	used_tools_must_exist(['cp','rm','strip','make','git','upx','cc','wc','tail','hyperfine'])
-	mut context := new_context()  
+	mut context := new_context()
 	mut fp := flag.new_flag_parser(os.args)
 	fp.application(os.filename(os.executable()))
 	fp.version( tool_version )
@@ -129,7 +174,7 @@ fn main(){
 	fp.arguments_description('COMMIT_BEFORE [COMMIT_AFTER]')
 	fp.skip_executable()
 	fp.limit_free_args(1,2)
-	show_help:=fp.bool('help',  false, 'Show this help screen')
+	show_help:=fp.bool('help', false, 'Show this help screen')
 	context.workdir = os.realpath( fp.string('workdir', '/tmp', 'A writable folder, where the comparison will be done.') )
 	if( show_help ){
 		println( fp.usage() )
@@ -139,21 +184,22 @@ fn main(){
 		eprintln('Error: ' + err)
 		exit(1)
 	}
-	
+
 	context.commit_before = commits[0]
 	if commits.len > 1 { context.commit_after = commits[1] }
-	
+
 	validate_commit_exists( context.commit_before )
 	validate_commit_exists( context.commit_after )
-	
-	context.b = context.normalized_workpath_for_commit( context.commit_before )
-	context.a = context.normalized_workpath_for_commit( context.commit_after )
-    
+
+	context.b  = context.normalized_workpath_for_commit( context.commit_before )
+	context.a  = context.normalized_workpath_for_commit( context.commit_after )
+	context.vc = context.normalized_workpath_for_commit( 'vc' )
+
 	if !os.is_dir( context.workdir ) {
 		msg := 'Work folder: ' + context.workdir + ' , does not exist.'
 		eprintln(msg)
 		exit(2)
 	}
-	
+
 	context.compare_versions()
 }