vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

builder: make -usecache rebuild cached, but changed modules, and their dependants (#12193)

pull/12205/head
Delyan Angelov 2021-10-15 12:22:59 +03:00 committed by GitHub
parent c108e01917
commit dee4ffbc99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 334 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
vlib/v/builder/builder.v
View File

@ -36,6 +36,11 @@ pub mut:
cached_msvc MsvcResult
table &ast.Table
ccoptions CcompilerOptions
//
// NB: changes in mod `builtin` force invalidation of every other .v file
mod_invalidates_paths map[string][]string // changes in mod `os`, invalidate only .v files, that do `import os`
mod_invalidates_mods map[string][]string // changes in mod `os`, force invalidation of mods, that do `import os`
path_invalidates_mods map[string][]string // changes in a .v file from `os`, invalidates `os`
}
pub fn new_builder(pref &pref.Preferences) Builder {
@ -150,8 +155,15 @@ pub fn (mut b Builder) parse_imports() {
// so we can not use the shorter `for in` form.
for i := 0; i < b.parsed_files.len; i++ {
ast_file := b.parsed_files[i]
b.path_invalidates_mods[ast_file.path] << ast_file.mod.name
if ast_file.mod.name != 'builtin' {
b.mod_invalidates_paths['builtin'] << ast_file.path
b.mod_invalidates_mods['builtin'] << ast_file.mod.name
}
for imp in ast_file.imports {
mod := imp.mod
b.mod_invalidates_paths[mod] << ast_file.path
b.mod_invalidates_mods[mod] << ast_file.mod.name
if mod == 'builtin' {
b.parsed_files[i].errors << b.error_with_pos('cannot import module "builtin"',
ast_file.path, imp.pos)
@ -174,6 +186,7 @@ pub fn (mut b Builder) parse_imports() {
ast_file.path, imp.pos)
continue
}
// eprintln('>> ast_file.path: $ast_file.path , done: $done_imports, `import $mod` => $v_files')
// Add all imports referenced by these libs
parsed_files := parser.parse_files(v_files, b.table, b.pref)
for file in parsed_files {
@ -192,13 +205,13 @@ pub fn (mut b Builder) parse_imports() {
}
}
b.resolve_deps()
//
if b.pref.print_v_files {
for p in b.parsed_files {
println(p.path)
}
exit(0)
}
b.rebuild_modules()
}
pub fn (mut b Builder) resolve_deps() {

97
vlib/v/builder/cc.v
View File

@ -120,28 +120,6 @@ fn (mut v Builder) post_process_c_compiler_output(res os.Result) {
verror(builder.c_error_info)
}
fn (mut v Builder) rebuild_cached_module(vexe string, imp_path string) string {
res := v.pref.cache_manager.exists('.o', imp_path) or {
if v.pref.is_verbose {
println('Cached $imp_path .o file not found... Building .o file for $imp_path')
}
// do run `v build-module x` always in main vfolder; x can be a relative path
pwd := os.getwd()
vroot := os.dir(vexe)
os.chdir(vroot) or {}
boptions := v.pref.build_options.join(' ')
rebuild_cmd := '$vexe $boptions build-module $imp_path'
vcache.dlog('| Builder.' + @FN, 'vexe: $vexe | imp_path: $imp_path | rebuild_cmd: $rebuild_cmd')
os.system(rebuild_cmd)
rebuilded_o := v.pref.cache_manager.exists('.o', imp_path) or {
panic('could not rebuild cache module for $imp_path, error: $err.msg')
}
os.chdir(pwd) or {}
return rebuilded_o
}
return res
}
fn (mut v Builder) show_cc(cmd string, response_file string, response_file_content string) {
if v.pref.is_verbose || v.pref.show_cc {
println('> C compiler cmd: $cmd')
@ -363,20 +341,7 @@ fn (mut v Builder) setup_ccompiler_options(ccompiler string) {
ccoptions.pre_args << defines
ccoptions.pre_args << others
ccoptions.linker_flags << libs
// TODO: why is this duplicated from above?
if v.pref.use_cache && v.pref.build_mode != .build_module {
// vexe := pref.vexe_path()
// cached_modules := ['builtin', 'os', 'math', 'strconv', 'strings', 'hash'], // , 'strconv.ftoa']
// for cfile in cached_modules {
// ofile := os.join_path(pref.default_module_path, 'cache', 'vlib', cfile.replace('.', '/') +
// '.o')
// if !os.exists(ofile) {
// println('${cfile}.o is missing. Building...')
// println('$vexe build-module vlib/$cfile')
// os.system('$vexe build-module vlib/$cfile')
// }
// args << ofile
// }
if !ccoptions.is_cc_tcc {
$if linux {
ccoptions.linker_flags << '-Xlinker -z'
@ -565,70 +530,10 @@ fn (mut v Builder) cc() {
}
}
}
//
mut libs := []string{} // builtin.o os.o http.o etc
if v.pref.build_mode == .build_module {
v.ccoptions.pre_args << '-c'
} else if v.pref.use_cache {
mut built_modules := []string{}
builtin_obj_path := v.rebuild_cached_module(vexe, 'vlib/builtin')
libs << builtin_obj_path
for ast_file in v.parsed_files {
if v.pref.is_test && ast_file.mod.name != 'main' {
imp_path := v.find_module_path(ast_file.mod.name, ast_file.path) or {
verror('cannot import module "$ast_file.mod.name" (not found)')
break
}
obj_path := v.rebuild_cached_module(vexe, imp_path)
libs << obj_path
built_modules << ast_file.mod.name
}
for imp_stmt in ast_file.imports {
imp := imp_stmt.mod
// strconv is already imported inside builtin, so skip generating its object file
// TODO: incase we have other modules with the same name, make sure they are vlib
// is this even doign anything?
if imp in ['strconv', 'strings'] {
continue
}
if imp in built_modules {
continue
}
if util.should_bundle_module(imp) {
continue
}
// not working
if imp == 'webview' {
continue
}
// The problem is cmd/v is in module main and imports
// the relative module named help, which is built as cmd.v.help not help
// currently this got this workign by building into main, see ast.FnDecl in cgen
if imp == 'help' {
continue
}
// we are skipping help manually above, this code will skip all relative imports
// if os.is_dir(af_base_dir + os.path_separator + mod_path) {
// continue
// }
// mod_path := imp.replace('.', os.path_separator)
// imp_path := os.join_path('vlib', mod_path)
imp_path := v.find_module_path(imp, ast_file.path) or {
verror('cannot import module "$imp" (not found)')
break
}
obj_path := v.rebuild_cached_module(vexe, imp_path)
libs << obj_path
if obj_path.ends_with('vlib/ui.o') {
v.ccoptions.post_args << '-framework Cocoa'
v.ccoptions.post_args << '-framework Carbon'
}
built_modules << imp
}
}
v.ccoptions.post_args << libs
}
//
v.handle_usecache(vexe)
if ccompiler == 'msvc' {
v.cc_msvc()
return

17
vlib/v/builder/compile.v
View File

@ -186,16 +186,6 @@ fn (mut v Builder) set_module_lookup_paths() {
}
pub fn (v Builder) get_builtin_files() []string {
/*
// if v.pref.build_mode == .build_module && v.pref.path == 'vlib/builtin' { // .contains('builtin/' + location {
if v.pref.build_mode == .build_module && v.pref.path == 'vlib/strconv' { // .contains('builtin/' + location {
// We are already building builtin.o, no need to import them again
if v.pref.is_verbose {
println('skipping builtin modules for builtin.o')
}
return []
}
*/
v.log('v.pref.lookup_path: $v.pref.lookup_path')
// Lookup for built-in folder in lookup path.
// Assumption: `builtin/` folder implies usable implementation of builtin
@ -225,7 +215,8 @@ pub fn (v Builder) get_builtin_files() []string {
}
pub fn (v &Builder) get_user_files() []string {
if v.pref.path in ['vlib/builtin', 'vlib/strconv', 'vlib/strings', 'vlib/hash'] {
if v.pref.path in ['vlib/builtin', 'vlib/strconv', 'vlib/strings', 'vlib/hash']
|| v.pref.path.ends_with('vlib/builtin') {
// This means we are building a builtin module with `v build-module vlib/strings` etc
// get_builtin_files() has already added the files in this module,
// do nothing here to avoid duplicate definition errors.
@ -267,8 +258,8 @@ pub fn (v &Builder) get_user_files() []string {
is_test := v.pref.is_test
mut is_internal_module_test := false
if is_test {
tcontent := os.read_file(dir) or { verror('$dir does not exist') }
slines := tcontent.trim_space().split_into_lines()
tcontent := util.read_file(dir) or { verror('$dir does not exist') }
slines := tcontent.split_into_lines()
for sline in slines {
line := sline.trim_space()
if line.len > 2 {

239
vlib/v/builder/rebuilding.v 100644
View File

@ -0,0 +1,239 @@
module builder
import os
import hash
import strings
import v.util
import v.pref
import v.vcache
pub fn (mut b Builder) rebuild_modules() {
if !b.pref.use_cache || b.pref.build_mode == .build_module {
return
}
util.timing_start('${@METHOD} source_hashing')
mut new_hashes := map[string]string{}
mut old_hashes := map[string]string{}
mut sb_new_hashes := strings.new_builder(1024)
all_files := b.parsed_files.map(it.path)
//
mut cm := vcache.new_cache_manager(all_files)
sold_hashes := cm.load('.hashes', 'all_files') or { ' ' }
// eprintln(sold_hashes)
sold_hashes_lines := sold_hashes.split('\n')
for line in sold_hashes_lines {
if line.len == 0 {
continue
}
x := line.split(' ')
chash := x[0]
cpath := x[1]
old_hashes[cpath] = chash
}
// eprintln('old_hashes: $old_hashes')
for p in b.parsed_files {
cpath := p.path
ccontent := util.read_file(cpath) or { '' }
chash := hash.sum64_string(ccontent, 7).hex_full()
new_hashes[cpath] = chash
sb_new_hashes.write_string(chash)
sb_new_hashes.write_b(` `)
sb_new_hashes.write_string(cpath)
sb_new_hashes.write_b(`\n`)
}
snew_hashes := sb_new_hashes.str()
// eprintln('new_hashes: $new_hashes')
// eprintln('> new_hashes != old_hashes: ' + ( old_hashes != new_hashes ).str())
// eprintln(snew_hashes)
cm.save('.hashes', 'all_files', snew_hashes) or {}
util.timing_measure('${@METHOD} source_hashing')
if new_hashes != old_hashes {
util.timing_start('${@METHOD} rebuilding')
// eprintln('> b.mod_invalidates_paths: $b.mod_invalidates_paths')
// eprintln('> b.mod_invalidates_mods: $b.mod_invalidates_mods')
// eprintln('> b.path_invalidates_mods: $b.path_invalidates_mods')
$if trace_invalidations ? {
for k, v in b.mod_invalidates_paths {
mut m := map[string]bool{}
for mm in b.mod_invalidates_mods[k] {
m[mm] = true
}
eprintln('> module `$k` invalidates: $m.keys()')
for fpath in v {
eprintln(' $fpath')
}
}
}
mut invalidated_paths := map[string]int{}
mut invalidated_mod_paths := map[string]int{}
for npath, nhash in new_hashes {
if npath !in old_hashes {
invalidated_paths[npath]++
continue
}
if old_hashes[npath] != nhash {
invalidated_paths[npath]++
continue
}
}
for opath, ohash in old_hashes {
if opath !in new_hashes {
invalidated_paths[opath]++
continue
}
if new_hashes[opath] != ohash {
invalidated_paths[opath]++
continue
}
}
$if trace_invalidations ? {
eprintln('invalidated_paths: $invalidated_paths')
}
mut rebuild_everything := false
for cycle := 0; true; cycle++ {
$if trace_invalidations ? {
eprintln('> cycle: $cycle | invalidated_paths: $invalidated_paths')
}
mut new_invalidated_paths := map[string]int{}
for npath, _ in invalidated_paths {
invalidated_mods := b.path_invalidates_mods[npath]
if invalidated_mods == ['main'] {
continue
}
if 'builtin' in invalidated_mods {
// When `builtin` is invalid, there is no point in
// extracting a finer grained dependency resolution
// of the dependencies any more. Instead, just rebuild
// every module.
rebuild_everything = true
break
}
for imod in invalidated_mods {
if imod == 'main' {
continue
}
for np in b.mod_invalidates_paths[imod] {
new_invalidated_paths[np]++
}
}
$if trace_invalidations ? {
eprintln('> npath -> invalidated_mods | $npath -> $invalidated_mods')
}
mpath := os.dir(npath)
invalidated_mod_paths[mpath]++
}
if rebuild_everything {
break
}
if new_invalidated_paths.len == 0 {
break
}
invalidated_paths = new_invalidated_paths.clone()
}
if rebuild_everything {
invalidated_mod_paths = {}
for npath, _ in new_hashes {
mpath := os.dir(npath)
pimods := b.path_invalidates_mods[npath]
if pimods == ['main'] {
continue
}
invalidated_mod_paths[mpath]++
}
}
$if trace_invalidations ? {
eprintln('invalidated_mod_paths: $invalidated_mod_paths')
eprintln('rebuild_everything: $rebuild_everything')
}
if invalidated_mod_paths.len > 0 {
impaths := invalidated_mod_paths.keys()
vexe := pref.vexe_path()
for imp in impaths {
b.v_build_module(vexe, imp)
}
}
util.timing_measure('${@METHOD} rebuilding')
}
}
fn (mut b Builder) v_build_module(vexe string, imp_path string) {
pwd := os.getwd()
defer {
os.chdir(pwd) or {}
}
// do run `v build-module x` always in main vfolder; x can be a relative path
vroot := os.dir(vexe)
os.chdir(vroot) or {}
boptions := b.pref.build_options.join(' ')
rebuild_cmd := '$vexe $boptions build-module $imp_path'
vcache.dlog('| Builder.' + @FN, 'vexe: $vexe | imp_path: $imp_path | rebuild_cmd: $rebuild_cmd')
$if trace_v_build_module ? {
eprintln('> Builder.v_build_module: $rebuild_cmd')
}
os.system(rebuild_cmd)
}
fn (mut b Builder) rebuild_cached_module(vexe string, imp_path string) string {
res := b.pref.cache_manager.exists('.o', imp_path) or {
if b.pref.is_verbose {
println('Cached $imp_path .o file not found... Building .o file for $imp_path')
}
b.v_build_module(vexe, imp_path)
rebuilded_o := b.pref.cache_manager.exists('.o', imp_path) or {
panic('could not rebuild cache module for $imp_path, error: $err.msg')
}
return rebuilded_o
}
return res
}
fn (mut b Builder) handle_usecache(vexe string) {
if !b.pref.use_cache || b.pref.build_mode == .build_module {
return
}
mut libs := []string{} // builtin.o os.o http.o etc
mut built_modules := []string{}
builtin_obj_path := b.rebuild_cached_module(vexe, 'vlib/builtin')
libs << builtin_obj_path
for ast_file in b.parsed_files {
if b.pref.is_test && ast_file.mod.name != 'main' {
imp_path := b.find_module_path(ast_file.mod.name, ast_file.path) or {
verror('cannot import module "$ast_file.mod.name" (not found)')
break
}
obj_path := b.rebuild_cached_module(vexe, imp_path)
libs << obj_path
built_modules << ast_file.mod.name
}
for imp_stmt in ast_file.imports {
imp := imp_stmt.mod
// strconv is already imported inside builtin, so skip generating its object file
// TODO: incase we have other modules with the same name, make sure they are vlib
// is this even doign anything?
if imp in ['strconv', 'strings'] {
continue
}
if imp in built_modules {
continue
}
if util.should_bundle_module(imp) {
continue
}
// The problem is cmd/v is in module main and imports
// the relative module named help, which is built as cmd.v.help not help
// currently this got this workign by building into main, see ast.FnDecl in cgen
if imp == 'help' {
continue
}
imp_path := b.find_module_path(imp, ast_file.path) or {
verror('cannot import module "$imp" (not found)')
break
}
obj_path := b.rebuild_cached_module(vexe, imp_path)
libs << obj_path
built_modules << imp
}
}
b.ccoptions.post_args << libs
}

1
vlib/v/tests/testdata/usecache_and_mods/aaa/aa1.v vendored 100644
View File

@ -0,0 +1 @@
module aaa

5
vlib/v/tests/testdata/usecache_and_mods/aaa/aaa.v vendored 100644
View File

@ -0,0 +1,5 @@
module aaa
import v.tests.testdata.usecache_and_mods.bbb
pub const used = bbb.used + 1

1
vlib/v/tests/testdata/usecache_and_mods/bbb/bb1.v vendored 100644
View File

@ -0,0 +1 @@
module bbb

5
vlib/v/tests/testdata/usecache_and_mods/bbb/bbb.v vendored 100644
View File

@ -0,0 +1,5 @@
module bbb
import v.tests.testdata.usecache_and_mods.ccc
pub const used = ccc.used

1
vlib/v/tests/testdata/usecache_and_mods/ccc/cc1.v vendored 100644
View File

@ -0,0 +1 @@
module ccc

1
vlib/v/tests/testdata/usecache_and_mods/ccc/cc2.v vendored 100644
View File

@ -0,0 +1 @@
module ccc

5
vlib/v/tests/testdata/usecache_and_mods/ccc/ccc.v vendored 100644
View File

@ -0,0 +1,5 @@
module ccc
import v.tests.testdata.usecache_and_mods.ddd
pub const used = ddd.used + 1

1
vlib/v/tests/testdata/usecache_and_mods/ddd/dd1.v vendored 100644
View File

@ -0,0 +1 @@
module ddd

3
vlib/v/tests/testdata/usecache_and_mods/ddd/ddd.v vendored 100644
View File

@ -0,0 +1,3 @@
module ddd
pub const used = 1

16
vlib/v/tests/testdata/usecache_and_mods/main.v vendored 100644
View File

@ -0,0 +1,16 @@
import v.tests.testdata.usecache_and_mods.xx
import v.tests.testdata.usecache_and_mods.aaa
import strconv
import strings
const used = aaa.used + xx.used
fn main() {
println(used)
println(strconv.c_ten)
mut sb := strings.new_builder(1024)
sb.writeln('hello')
sb.writeln('world')
print(sb.str())
println('----- done ----')
}

0
vlib/v/tests/testdata/usecache_and_mods/v.mod vendored 100644
View File

1
vlib/v/tests/testdata/usecache_and_mods/xx/x1.v vendored 100644
View File

@ -0,0 +1 @@
module xx

5
vlib/v/tests/testdata/usecache_and_mods/xx/xx.v vendored 100644
View File

@ -0,0 +1,5 @@
module xx
import v.tests.testdata.usecache_and_mods.yy
pub const used = yy.used + 1

1
vlib/v/tests/testdata/usecache_and_mods/yy/y1.v vendored 100644
View File

@ -0,0 +1 @@
module yy

5
vlib/v/tests/testdata/usecache_and_mods/yy/yy.v vendored 100644
View File

@ -0,0 +1,5 @@
module yy
import v.tests.testdata.usecache_and_mods.zz
pub const used = zz.used + 1

1
vlib/v/tests/testdata/usecache_and_mods/zz/z1.v vendored 100644
View File

@ -0,0 +1 @@
module zz

3
vlib/v/tests/testdata/usecache_and_mods/zz/zz.v vendored 100644
View File

@ -0,0 +1,3 @@
module zz
pub const used = 100

30
vlib/v/vcache/vcache.v
View File

@ -42,12 +42,17 @@ pub fn new_cache_manager(opts []string) CacheManager {
dlog(@FN, 'vcache_basepath: $vcache_basepath | opts:\n $opts')
if !os.is_dir(vcache_basepath) {
os.mkdir_all(vcache_basepath) or { panic(err) }
dlog(@FN, 'created folder:\n $vcache_basepath')
}
readme_file := os.join_path(vcache_basepath, 'README.md')
if !os.is_file(readme_file) {
readme_content := 'This folder contains cached build artifacts from the V build system.
|You can safely delete it, if it is getting too large.
|It will be recreated the next time you compile something with V.
|You can change its location with the VCACHE environment variable.
'.strip_margin()
os.write_file(os.join_path(vcache_basepath, 'README.md'), readme_content) or { panic(err) }
os.write_file(readme_file, readme_content) or { panic(err) }
dlog(@FN, 'created readme_file:\n $readme_file')
}
original_vopts := opts.join('|')
return CacheManager{
@ -118,14 +123,21 @@ pub fn (mut cm CacheManager) load(postfix string, key string) ?string {
return content
}
const process_pid = os.getpid()
[if trace_use_cache ?]
pub fn dlog(fname string, s string) {
$if trace_use_cache ? {
if fname[0] != `|` {
eprintln('> VCache | pid: $vcache.process_pid | CacheManager.$fname $s')
} else {
eprintln('> VCache | pid: $vcache.process_pid $fname $s')
}
pid := unsafe { mypid() }
if fname[0] != `|` {
eprintln('> VCache | pid: $pid | CacheManager.$fname $s')
} else {
eprintln('> VCache | pid: $pid $fname $s')
}
}
[unsafe]
fn mypid() int {
mut static pid := 0
if pid == 0 {
pid = os.getpid()
}
return pid
}