builder: make -usecache rebuild cached, but changed modules, and their dependants (#12193)

vlib/v/builder/builder.v

@@ -36,6 +36,11 @@ pub mut:
 	cached_msvc         MsvcResult
 	table               &ast.Table
 	ccoptions           CcompilerOptions
+	//
+	// NB: changes in mod `builtin` force invalidation of every other .v file
+	mod_invalidates_paths map[string][]string // changes in mod `os`, invalidate only .v files, that do `import os`
+	mod_invalidates_mods  map[string][]string // changes in mod `os`, force invalidation of mods, that do `import os`
+	path_invalidates_mods map[string][]string // changes in a .v file from `os`, invalidates `os`
 }
 
 pub fn new_builder(pref &pref.Preferences) Builder {
@@ -150,8 +155,15 @@ pub fn (mut b Builder) parse_imports() {
 	// so we can not use the shorter `for in` form.
 	for i := 0; i < b.parsed_files.len; i++ {
 		ast_file := b.parsed_files[i]
+		b.path_invalidates_mods[ast_file.path] << ast_file.mod.name
+		if ast_file.mod.name != 'builtin' {
+			b.mod_invalidates_paths['builtin'] << ast_file.path
+			b.mod_invalidates_mods['builtin'] << ast_file.mod.name
+		}
 		for imp in ast_file.imports {
 			mod := imp.mod
+			b.mod_invalidates_paths[mod] << ast_file.path
+			b.mod_invalidates_mods[mod] << ast_file.mod.name
 			if mod == 'builtin' {
 				b.parsed_files[i].errors << b.error_with_pos('cannot import module "builtin"',
 					ast_file.path, imp.pos)
@@ -174,6 +186,7 @@ pub fn (mut b Builder) parse_imports() {
 					ast_file.path, imp.pos)
 				continue
 			}
+			// eprintln('>> ast_file.path: $ast_file.path , done: $done_imports, `import $mod` => $v_files')
 			// Add all imports referenced by these libs
 			parsed_files := parser.parse_files(v_files, b.table, b.pref)
 			for file in parsed_files {
@@ -192,13 +205,13 @@ pub fn (mut b Builder) parse_imports() {
 		}
 	}
 	b.resolve_deps()
-	//
 	if b.pref.print_v_files {
 		for p in b.parsed_files {
 			println(p.path)
 		}
 		exit(0)
 	}
+	b.rebuild_modules()
 }
 
 pub fn (mut b Builder) resolve_deps() {

vlib/v/builder/cc.v

@@ -120,28 +120,6 @@ fn (mut v Builder) post_process_c_compiler_output(res os.Result) {
 	verror(builder.c_error_info)
 }
 
-fn (mut v Builder) rebuild_cached_module(vexe string, imp_path string) string {
-	res := v.pref.cache_manager.exists('.o', imp_path) or {
-		if v.pref.is_verbose {
-			println('Cached $imp_path .o file not found... Building .o file for $imp_path')
-		}
-		// do run `v build-module x` always in main vfolder; x can be a relative path
-		pwd := os.getwd()
-		vroot := os.dir(vexe)
-		os.chdir(vroot) or {}
-		boptions := v.pref.build_options.join(' ')
-		rebuild_cmd := '$vexe $boptions build-module $imp_path'
-		vcache.dlog('| Builder.' + @FN, 'vexe: $vexe | imp_path: $imp_path | rebuild_cmd: $rebuild_cmd')
-		os.system(rebuild_cmd)
-		rebuilded_o := v.pref.cache_manager.exists('.o', imp_path) or {
-			panic('could not rebuild cache module for $imp_path, error: $err.msg')
-		}
-		os.chdir(pwd) or {}
-		return rebuilded_o
-	}
-	return res
-}
-
 fn (mut v Builder) show_cc(cmd string, response_file string, response_file_content string) {
 	if v.pref.is_verbose || v.pref.show_cc {
 		println('> C compiler cmd: $cmd')
@@ -363,20 +341,7 @@ fn (mut v Builder) setup_ccompiler_options(ccompiler string) {
 	ccoptions.pre_args << defines
 	ccoptions.pre_args << others
 	ccoptions.linker_flags << libs
-	// TODO: why is this duplicated from above?
 	if v.pref.use_cache && v.pref.build_mode != .build_module {
-		// vexe := pref.vexe_path()
-		// cached_modules := ['builtin', 'os', 'math', 'strconv', 'strings', 'hash'],  // , 'strconv.ftoa']
-		// for cfile in cached_modules {
-		// ofile := os.join_path(pref.default_module_path, 'cache', 'vlib', cfile.replace('.', '/') +
-		// '.o')
-		// if !os.exists(ofile) {
-		// println('${cfile}.o is missing. Building...')
-		// println('$vexe build-module vlib/$cfile')
-		// os.system('$vexe build-module vlib/$cfile')
-		// }
-		// args << ofile
-		// }
 		if !ccoptions.is_cc_tcc {
 			$if linux {
 				ccoptions.linker_flags << '-Xlinker -z'
@@ -565,70 +530,10 @@ fn (mut v Builder) cc() {
 				}
 			}
 		}
-		//
-		mut libs := []string{} // builtin.o os.o http.o etc
 		if v.pref.build_mode == .build_module {
 			v.ccoptions.pre_args << '-c'
-		} else if v.pref.use_cache {
-			mut built_modules := []string{}
-			builtin_obj_path := v.rebuild_cached_module(vexe, 'vlib/builtin')
-			libs << builtin_obj_path
-			for ast_file in v.parsed_files {
-				if v.pref.is_test && ast_file.mod.name != 'main' {
-					imp_path := v.find_module_path(ast_file.mod.name, ast_file.path) or {
-						verror('cannot import module "$ast_file.mod.name" (not found)')
-						break
-					}
-					obj_path := v.rebuild_cached_module(vexe, imp_path)
-					libs << obj_path
-					built_modules << ast_file.mod.name
-				}
-				for imp_stmt in ast_file.imports {
-					imp := imp_stmt.mod
-					// strconv is already imported inside builtin, so skip generating its object file
-					// TODO: incase we have other modules with the same name, make sure they are vlib
-					// is this even doign anything?
-					if imp in ['strconv', 'strings'] {
-						continue
-					}
-					if imp in built_modules {
-						continue
-					}
-					if util.should_bundle_module(imp) {
-						continue
-					}
-					// not working
-					if imp == 'webview' {
-						continue
-					}
-					// The problem is cmd/v is in module main and imports
-					// the relative module named help, which is built as cmd.v.help not help
-					// currently this got this workign by building into main, see ast.FnDecl in cgen
-					if imp == 'help' {
-						continue
-					}
-					// we are skipping help manually above, this code will skip all relative imports
-					// if os.is_dir(af_base_dir + os.path_separator + mod_path) {
-					// continue
-					// }
-					// mod_path := imp.replace('.', os.path_separator)
-					// imp_path := os.join_path('vlib', mod_path)
-					imp_path := v.find_module_path(imp, ast_file.path) or {
-						verror('cannot import module "$imp" (not found)')
-						break
-					}
-					obj_path := v.rebuild_cached_module(vexe, imp_path)
-					libs << obj_path
-					if obj_path.ends_with('vlib/ui.o') {
-						v.ccoptions.post_args << '-framework Cocoa'
-						v.ccoptions.post_args << '-framework Carbon'
-					}
-					built_modules << imp
-				}
-			}
-			v.ccoptions.post_args << libs
 		}
-		//
+		v.handle_usecache(vexe)
 		if ccompiler == 'msvc' {
 			v.cc_msvc()
 			return

vlib/v/builder/compile.v

@@ -186,16 +186,6 @@ fn (mut v Builder) set_module_lookup_paths() {
 }
 
 pub fn (v Builder) get_builtin_files() []string {
-	/*
-	// if v.pref.build_mode == .build_module && v.pref.path == 'vlib/builtin' { // .contains('builtin/' +  location {
-	if v.pref.build_mode == .build_module && v.pref.path == 'vlib/strconv' { // .contains('builtin/' +  location {
-		// We are already building builtin.o, no need to import them again
-		if v.pref.is_verbose {
-			println('skipping builtin modules for builtin.o')
-		}
-		return []
-	}
-	*/
 	v.log('v.pref.lookup_path: $v.pref.lookup_path')
 	// Lookup for built-in folder in lookup path.
 	// Assumption: `builtin/` folder implies usable implementation of builtin
@@ -225,7 +215,8 @@ pub fn (v Builder) get_builtin_files() []string {
 }
 
 pub fn (v &Builder) get_user_files() []string {
-	if v.pref.path in ['vlib/builtin', 'vlib/strconv', 'vlib/strings', 'vlib/hash'] {
+	if v.pref.path in ['vlib/builtin', 'vlib/strconv', 'vlib/strings', 'vlib/hash']
+		|| v.pref.path.ends_with('vlib/builtin') {
 		// This means we are building a builtin module with `v build-module vlib/strings` etc
 		// get_builtin_files() has already added the files in this module,
 		// do nothing here to avoid duplicate definition errors.
@@ -267,8 +258,8 @@ pub fn (v &Builder) get_user_files() []string {
 	is_test := v.pref.is_test
 	mut is_internal_module_test := false
 	if is_test {
-		tcontent := os.read_file(dir) or { verror('$dir does not exist') }
+		tcontent := util.read_file(dir) or { verror('$dir does not exist') }
-		slines := tcontent.trim_space().split_into_lines()
+		slines := tcontent.split_into_lines()
 		for sline in slines {
 			line := sline.trim_space()
 			if line.len > 2 {

vlib/v/builder/rebuilding.v

@@ -0,0 +1,239 @@
+module builder
+
+import os
+import hash
+import strings
+import v.util
+import v.pref
+import v.vcache
+
+pub fn (mut b Builder) rebuild_modules() {
+	if !b.pref.use_cache || b.pref.build_mode == .build_module {
+		return
+	}
+	util.timing_start('${@METHOD} source_hashing')
+	mut new_hashes := map[string]string{}
+	mut old_hashes := map[string]string{}
+	mut sb_new_hashes := strings.new_builder(1024)
+	all_files := b.parsed_files.map(it.path)
+	//
+	mut cm := vcache.new_cache_manager(all_files)
+	sold_hashes := cm.load('.hashes', 'all_files') or { ' ' }
+	// eprintln(sold_hashes)
+	sold_hashes_lines := sold_hashes.split('\n')
+	for line in sold_hashes_lines {
+		if line.len == 0 {
+			continue
+		}
+		x := line.split(' ')
+		chash := x[0]
+		cpath := x[1]
+		old_hashes[cpath] = chash
+	}
+	// eprintln('old_hashes: $old_hashes')
+	for p in b.parsed_files {
+		cpath := p.path
+		ccontent := util.read_file(cpath) or { '' }
+		chash := hash.sum64_string(ccontent, 7).hex_full()
+		new_hashes[cpath] = chash
+		sb_new_hashes.write_string(chash)
+		sb_new_hashes.write_b(` `)
+		sb_new_hashes.write_string(cpath)
+		sb_new_hashes.write_b(`\n`)
+	}
+	snew_hashes := sb_new_hashes.str()
+	// eprintln('new_hashes: $new_hashes')
+	// eprintln('> new_hashes != old_hashes: ' + ( old_hashes != new_hashes ).str())
+	// eprintln(snew_hashes)
+	cm.save('.hashes', 'all_files', snew_hashes) or {}
+	util.timing_measure('${@METHOD} source_hashing')
+
+	if new_hashes != old_hashes {
+		util.timing_start('${@METHOD} rebuilding')
+		// eprintln('> b.mod_invalidates_paths: $b.mod_invalidates_paths')
+		// eprintln('> b.mod_invalidates_mods: $b.mod_invalidates_mods')
+		// eprintln('> b.path_invalidates_mods: $b.path_invalidates_mods')
+		$if trace_invalidations ? {
+			for k, v in b.mod_invalidates_paths {
+				mut m := map[string]bool{}
+				for mm in b.mod_invalidates_mods[k] {
+					m[mm] = true
+				}
+				eprintln('> module `$k` invalidates: $m.keys()')
+				for fpath in v {
+					eprintln('         $fpath')
+				}
+			}
+		}
+		mut invalidated_paths := map[string]int{}
+		mut invalidated_mod_paths := map[string]int{}
+		for npath, nhash in new_hashes {
+			if npath !in old_hashes {
+				invalidated_paths[npath]++
+				continue
+			}
+			if old_hashes[npath] != nhash {
+				invalidated_paths[npath]++
+				continue
+			}
+		}
+		for opath, ohash in old_hashes {
+			if opath !in new_hashes {
+				invalidated_paths[opath]++
+				continue
+			}
+			if new_hashes[opath] != ohash {
+				invalidated_paths[opath]++
+				continue
+			}
+		}
+		$if trace_invalidations ? {
+			eprintln('invalidated_paths: $invalidated_paths')
+		}
+		mut rebuild_everything := false
+		for cycle := 0; true; cycle++ {
+			$if trace_invalidations ? {
+				eprintln('> cycle: $cycle | invalidated_paths: $invalidated_paths')
+			}
+			mut new_invalidated_paths := map[string]int{}
+			for npath, _ in invalidated_paths {
+				invalidated_mods := b.path_invalidates_mods[npath]
+				if invalidated_mods == ['main'] {
+					continue
+				}
+				if 'builtin' in invalidated_mods {
+					// When `builtin` is invalid, there is no point in
+					// extracting a finer grained dependency resolution
+					// of the dependencies any more. Instead, just rebuild
+					// every module.
+					rebuild_everything = true
+					break
+				}
+				for imod in invalidated_mods {
+					if imod == 'main' {
+						continue
+					}
+					for np in b.mod_invalidates_paths[imod] {
+						new_invalidated_paths[np]++
+					}
+				}
+				$if trace_invalidations ? {
+					eprintln('> npath -> invalidated_mods | $npath -> $invalidated_mods')
+				}
+				mpath := os.dir(npath)
+				invalidated_mod_paths[mpath]++
+			}
+			if rebuild_everything {
+				break
+			}
+			if new_invalidated_paths.len == 0 {
+				break
+			}
+			invalidated_paths = new_invalidated_paths.clone()
+		}
+		if rebuild_everything {
+			invalidated_mod_paths = {}
+			for npath, _ in new_hashes {
+				mpath := os.dir(npath)
+				pimods := b.path_invalidates_mods[npath]
+				if pimods == ['main'] {
+					continue
+				}
+				invalidated_mod_paths[mpath]++
+			}
+		}
+		$if trace_invalidations ? {
+			eprintln('invalidated_mod_paths: $invalidated_mod_paths')
+			eprintln('rebuild_everything: $rebuild_everything')
+		}
+		if invalidated_mod_paths.len > 0 {
+			impaths := invalidated_mod_paths.keys()
+			vexe := pref.vexe_path()
+			for imp in impaths {
+				b.v_build_module(vexe, imp)
+			}
+		}
+		util.timing_measure('${@METHOD} rebuilding')
+	}
+}
+
+fn (mut b Builder) v_build_module(vexe string, imp_path string) {
+	pwd := os.getwd()
+	defer {
+		os.chdir(pwd) or {}
+	}
+	// do run `v build-module x` always in main vfolder; x can be a relative path
+	vroot := os.dir(vexe)
+	os.chdir(vroot) or {}
+	boptions := b.pref.build_options.join(' ')
+	rebuild_cmd := '$vexe $boptions build-module $imp_path'
+	vcache.dlog('| Builder.' + @FN, 'vexe: $vexe | imp_path: $imp_path | rebuild_cmd: $rebuild_cmd')
+	$if trace_v_build_module ? {
+		eprintln('> Builder.v_build_module: $rebuild_cmd')
+	}
+	os.system(rebuild_cmd)
+}
+
+fn (mut b Builder) rebuild_cached_module(vexe string, imp_path string) string {
+	res := b.pref.cache_manager.exists('.o', imp_path) or {
+		if b.pref.is_verbose {
+			println('Cached $imp_path .o file not found... Building .o file for $imp_path')
+		}
+		b.v_build_module(vexe, imp_path)
+		rebuilded_o := b.pref.cache_manager.exists('.o', imp_path) or {
+			panic('could not rebuild cache module for $imp_path, error: $err.msg')
+		}
+		return rebuilded_o
+	}
+	return res
+}
+
+fn (mut b Builder) handle_usecache(vexe string) {
+	if !b.pref.use_cache || b.pref.build_mode == .build_module {
+		return
+	}
+	mut libs := []string{} // builtin.o os.o http.o etc
+	mut built_modules := []string{}
+	builtin_obj_path := b.rebuild_cached_module(vexe, 'vlib/builtin')
+	libs << builtin_obj_path
+	for ast_file in b.parsed_files {
+		if b.pref.is_test && ast_file.mod.name != 'main' {
+			imp_path := b.find_module_path(ast_file.mod.name, ast_file.path) or {
+				verror('cannot import module "$ast_file.mod.name" (not found)')
+				break
+			}
+			obj_path := b.rebuild_cached_module(vexe, imp_path)
+			libs << obj_path
+			built_modules << ast_file.mod.name
+		}
+		for imp_stmt in ast_file.imports {
+			imp := imp_stmt.mod
+			// strconv is already imported inside builtin, so skip generating its object file
+			// TODO: incase we have other modules with the same name, make sure they are vlib
+			// is this even doign anything?
+			if imp in ['strconv', 'strings'] {
+				continue
+			}
+			if imp in built_modules {
+				continue
+			}
+			if util.should_bundle_module(imp) {
+				continue
+			}
+			// The problem is cmd/v is in module main and imports
+			// the relative module named help, which is built as cmd.v.help not help
+			// currently this got this workign by building into main, see ast.FnDecl in cgen
+			if imp == 'help' {
+				continue
+			}
+			imp_path := b.find_module_path(imp, ast_file.path) or {
+				verror('cannot import module "$imp" (not found)')
+				break
+			}
+			obj_path := b.rebuild_cached_module(vexe, imp_path)
+			libs << obj_path
+			built_modules << imp
+		}
+	}
+	b.ccoptions.post_args << libs
+}

vlib/v/tests/testdata/usecache_and_mods/aaa/aa1.v

@@ -0,0 +1 @@
+module aaa

vlib/v/tests/testdata/usecache_and_mods/aaa/aaa.v

@@ -0,0 +1,5 @@
+module aaa
+
+import v.tests.testdata.usecache_and_mods.bbb
+
+pub const used = bbb.used + 1

vlib/v/tests/testdata/usecache_and_mods/bbb/bb1.v

@@ -0,0 +1 @@
+module bbb

vlib/v/tests/testdata/usecache_and_mods/bbb/bbb.v

@@ -0,0 +1,5 @@
+module bbb
+
+import v.tests.testdata.usecache_and_mods.ccc
+
+pub const used = ccc.used

vlib/v/tests/testdata/usecache_and_mods/ccc/cc1.v

@@ -0,0 +1 @@
+module ccc

vlib/v/tests/testdata/usecache_and_mods/ccc/cc2.v

@@ -0,0 +1 @@
+module ccc

vlib/v/tests/testdata/usecache_and_mods/ccc/ccc.v

@@ -0,0 +1,5 @@
+module ccc
+
+import v.tests.testdata.usecache_and_mods.ddd
+
+pub const used = ddd.used + 1

vlib/v/tests/testdata/usecache_and_mods/ddd/dd1.v

@@ -0,0 +1 @@
+module ddd

vlib/v/tests/testdata/usecache_and_mods/ddd/ddd.v

@@ -0,0 +1,3 @@
+module ddd
+
+pub const used = 1

vlib/v/tests/testdata/usecache_and_mods/main.v

@@ -0,0 +1,16 @@
+import v.tests.testdata.usecache_and_mods.xx
+import v.tests.testdata.usecache_and_mods.aaa
+import strconv
+import strings
+
+const used = aaa.used + xx.used
+
+fn main() {
+	println(used)
+	println(strconv.c_ten)
+	mut sb := strings.new_builder(1024)
+	sb.writeln('hello')
+	sb.writeln('world')
+	print(sb.str())
+	println('----- done ----')
+}

vlib/v/tests/testdata/usecache_and_mods/xx/x1.v

@@ -0,0 +1 @@
+module xx

vlib/v/tests/testdata/usecache_and_mods/xx/xx.v

@@ -0,0 +1,5 @@
+module xx
+
+import v.tests.testdata.usecache_and_mods.yy
+
+pub const used = yy.used + 1

vlib/v/tests/testdata/usecache_and_mods/yy/y1.v

@@ -0,0 +1 @@
+module yy

vlib/v/tests/testdata/usecache_and_mods/yy/yy.v

@@ -0,0 +1,5 @@
+module yy
+
+import v.tests.testdata.usecache_and_mods.zz
+
+pub const used = zz.used + 1

vlib/v/tests/testdata/usecache_and_mods/zz/z1.v

@@ -0,0 +1 @@
+module zz

vlib/v/tests/testdata/usecache_and_mods/zz/zz.v

@@ -0,0 +1,3 @@
+module zz
+
+pub const used = 100

vlib/v/vcache/vcache.v

@@ -42,12 +42,17 @@ pub fn new_cache_manager(opts []string) CacheManager {
 	dlog(@FN, 'vcache_basepath: $vcache_basepath | opts:\n     $opts')
 	if !os.is_dir(vcache_basepath) {
 		os.mkdir_all(vcache_basepath) or { panic(err) }
+		dlog(@FN, 'created folder:\n    $vcache_basepath')
+	}
+	readme_file := os.join_path(vcache_basepath, 'README.md')
+	if !os.is_file(readme_file) {
 		readme_content := 'This folder contains cached build artifacts from the V build system.
 		|You can safely delete it, if it is getting too large.
 		|It will be recreated the next time you compile something with V.
 		|You can change its location with the VCACHE environment variable.
 		'.strip_margin()
-		os.write_file(os.join_path(vcache_basepath, 'README.md'), readme_content) or { panic(err) }
+		os.write_file(readme_file, readme_content) or { panic(err) }
+		dlog(@FN, 'created readme_file:\n    $readme_file')
 	}
 	original_vopts := opts.join('|')
 	return CacheManager{
@@ -118,14 +123,21 @@ pub fn (mut cm CacheManager) load(postfix string, key string) ?string {
 	return content
 }
 
-const process_pid = os.getpid()
+[if trace_use_cache ?]
-
 pub fn dlog(fname string, s string) {
-	$if trace_use_cache ? {
+	pid := unsafe { mypid() }
-		if fname[0] != `|` {
+	if fname[0] != `|` {
-			eprintln('> VCache | pid: $vcache.process_pid | CacheManager.$fname $s')
+		eprintln('> VCache | pid: $pid | CacheManager.$fname $s')
-		} else {
+	} else {
-			eprintln('> VCache | pid: $vcache.process_pid $fname $s')
+		eprintln('> VCache | pid: $pid $fname $s')
-		}
 	}
 }
+
+[unsafe]
+fn mypid() int {
+	mut static pid := 0
+	if pid == 0 {
+		pid = os.getpid()
+	}
+	return pid
+}