diff --git a/vlib/v/builder/cc.v b/vlib/v/builder/cc.v index 6ed7401406..6e0161c75c 100644 --- a/vlib/v/builder/cc.v +++ b/vlib/v/builder/cc.v @@ -118,6 +118,21 @@ fn (mut v Builder) post_process_c_compiler_output(res os.Result) { verror(c_error_info) } +fn (mut v Builder) rebuild_cached_module(vexe string, imp_path string) string { + res := v.pref.cache_manager.exists('.o', imp_path) or { + println('Cached $imp_path .o file not found... Building .o file for $imp_path') + boptions := v.pref.build_options.join(' ') + rebuild_cmd := '$vexe $boptions build-module $imp_path' + // eprintln('>> rebuild_cmd: $rebuild_cmd') + os.system(rebuild_cmd) + rebuilded_o := v.pref.cache_manager.exists('.o', imp_path) or { + panic('could not rebuild cache module for $imp_path, error: $err') + } + return rebuilded_o + } + return res +} + fn (mut v Builder) cc() { if os.executable().contains('vfmt') { return @@ -260,14 +275,9 @@ fn (mut v Builder) cc() { linker_flags << '-nostdlib' } if v.pref.build_mode == .build_module { - // Create the modules & out directory if it's not there. - out_dir := os.join_path(pref.default_module_path, 'cache', v.pref.path) - pdir := out_dir.all_before_last(os.path_separator) - if !os.is_dir(pdir) { - os.mkdir_all(pdir) - } - v.pref.out_name = '${out_dir}.o' // v.out_name - println('Building ${v.pref.out_name}...') + v.pref.out_name = v.pref.cache_manager.postfix_with_key2cpath('.o', v.pref.path) // v.out_name + println('Building $v.pref.path to $v.pref.out_name ...') + v.pref.cache_manager.save('.description.txt', v.pref.path, '${v.pref.path:-30} @ $v.pref.cache_manager.vopts\n') // println('v.table.imports:') // println(v.table.imports) } @@ -349,10 +359,7 @@ fn (mut v Builder) cc() { args << '-c' } else if v.pref.use_cache { mut built_modules := []string{} - builtin_obj_path := os.join_path(pref.default_module_path, 'cache', 'vlib', 'builtin.o') - if !os.exists(builtin_obj_path) { - os.system('$vexe build-module vlib/builtin') - } + builtin_obj_path := v.rebuild_cached_module(vexe, 'vlib/builtin') libs += ' ' + builtin_obj_path for ast_file in v.parsed_files { for imp_stmt in ast_file.imports { @@ -382,14 +389,8 @@ fn (mut v Builder) cc() { // continue // } imp_path := os.join_path('vlib', mod_path) - cache_path := os.join_path(pref.default_module_path, 'cache') - obj_path := os.join_path(cache_path, '${imp_path}.o') - if os.exists(obj_path) { - libs += ' ' + obj_path - } else { - println('$obj_path not found... building module $imp') - os.system('$vexe build-module $imp_path') - } + obj_path := v.rebuild_cached_module(vexe, imp_path) + libs += ' ' + obj_path if obj_path.ends_with('vlib/ui.o') { args << '-framework Cocoa -framework Carbon' } @@ -777,21 +778,19 @@ fn (mut v Builder) build_thirdparty_obj_file(path string, moduleflags []cflag.CF if v.pref.os == .windows { // Cross compiling for Windows $if !windows { - if os.exists(obj_path) { - os.rm(obj_path) - } v.pref.ccompiler = mingw_cc } } - if os.exists(obj_path) { + opath := v.pref.cache_manager.postfix_with_key2cpath('.o', obj_path) + if os.exists(opath) { return } - println('$obj_path not found, building it...') + println('$obj_path not found, building it in $opath ...') cfile := '${path[..path.len - 2]}.c' btarget := moduleflags.c_options_before_target() atarget := moduleflags.c_options_after_target() cppoptions := if v.pref.ccompiler.contains('++') { ' -fpermissive -w ' } else { '' } - cmd := '$v.pref.ccompiler $cppoptions $v.pref.third_party_option $btarget -c -o "$obj_path" "$cfile" $atarget' + cmd := '$v.pref.ccompiler $cppoptions $v.pref.third_party_option $btarget -c -o "$opath" "$cfile" $atarget' res := os.exec(cmd) or { eprintln('exec failed for thirdparty object build cmd:\n$cmd') verror(err) @@ -802,6 +801,7 @@ fn (mut v Builder) build_thirdparty_obj_file(path string, moduleflags []cflag.CF verror(res.output) return } + v.pref.cache_manager.save('.description.txt', obj_path, '${obj_path:-30} @ $cmd\n') println(res.output) } diff --git a/vlib/v/builder/cflags.v b/vlib/v/builder/cflags.v index 139f75d0e2..b425046563 100644 --- a/vlib/v/builder/cflags.v +++ b/vlib/v/builder/cflags.v @@ -1,15 +1,19 @@ module builder +import os import v.cflag // get flags for current os -fn (v &Builder) get_os_cflags() []cflag.CFlag { +fn (mut v Builder) get_os_cflags() []cflag.CFlag { mut flags := []cflag.CFlag{} mut ctimedefines := []string{} if v.pref.compile_defines.len > 0 { ctimedefines << v.pref.compile_defines } - for flag in v.table.cflags { + for mut flag in v.table.cflags { + if flag.value.ends_with('.o') { + flag.cached = v.pref.cache_manager.postfix_with_key2cpath('.o', os.real_path(flag.value)) + } if flag.os == '' || (flag.os == 'linux' && v.pref.os == .linux) || (flag.os == 'macos' && v.pref.os == .macos) || @@ -27,7 +31,7 @@ fn (v &Builder) get_os_cflags() []cflag.CFlag { return flags } -fn (v &Builder) get_rest_of_module_cflags(c &cflag.CFlag) []cflag.CFlag { +fn (mut v Builder) get_rest_of_module_cflags(c &cflag.CFlag) []cflag.CFlag { mut flags := []cflag.CFlag{} cflags := v.get_os_cflags() for flag in cflags { diff --git a/vlib/v/cflag/cflags.v b/vlib/v/cflag/cflags.v index 54a6d8ca63..1bc49705de 100644 --- a/vlib/v/cflag/cflags.v +++ b/vlib/v/cflag/cflags.v @@ -8,19 +8,24 @@ import os // parsed cflag pub struct CFlag { pub: - mod string // the module in which the flag was given - os string // eg. windows | darwin | linux - name string // eg. -I - value string // eg. /path/to/include + mod string // the module in which the flag was given + os string // eg. windows | darwin | linux + name string // eg. -I + value string // eg. /path/to/include +pub mut: + cached string // eg. ~/.vmodules/cache/ea/ea9878886727367672163.o (for .o files) } pub fn (c &CFlag) str() string { - return 'CFlag{ name: "$c.name" value: "$c.value" mod: "$c.mod" os: "$c.os" }' + return 'CFlag{ name: "$c.name" value: "$c.value" mod: "$c.mod" os: "$c.os" cached: "$c.cached" }' } // format flag pub fn (cf &CFlag) format() string { mut value := cf.value + if cf.cached != '' { + value = cf.cached + } if cf.name in ['-l', '-Wa', '-Wl', '-Wp'] && value.len > 0 { return '$cf.name$value'.trim_space() } diff --git a/vlib/v/pref/default.v b/vlib/v/pref/default.v index f4a3428fcb..5371a56ff5 100644 --- a/vlib/v/pref/default.v +++ b/vlib/v/pref/default.v @@ -71,6 +71,17 @@ pub fn (mut p Preferences) fill_with_defaults() { } } } + // Prepare the cache manager. All options that can affect the generated cached .c files + // should go into res.cache_manager.vopts, which is used as a salt for the cache hash. + p.cache_manager = new_cache_manager([ + '$p.backend | $p.os | $p.ccompiler', + p.cflags.trim_space(), + p.third_party_option.trim_space(), + '$p.compile_defines_all', + '$p.compile_defines', + '$p.lookup_path', + ]) + // eprintln('prefs.cache_manager: $p') } fn default_c_compiler() string { diff --git a/vlib/v/pref/pref.v b/vlib/v/pref/pref.v index 9a12ba679b..f9a00011e9 100644 --- a/vlib/v/pref/pref.v +++ b/vlib/v/pref/pref.v @@ -129,6 +129,8 @@ pub mut: is_ios_simulator bool is_apk bool // build as Android .apk format cleanup_files []string // list of temporary *.tmp.c and *.tmp.c.rsp files. Cleaned up on successfull builds. + build_options []string // list of options, that should be passed down to `build-module`, if needed for -usecache + cache_manager CacheManager } pub fn parse_args(args []string) (&Preferences, string) { @@ -142,6 +144,7 @@ pub fn parse_args(args []string) (&Preferences, string) { match arg { '-apk' { res.is_apk = true + res.build_options << arg } '-show-timings' { res.show_timings = true @@ -167,10 +170,12 @@ pub fn parse_args(args []string) (&Preferences, string) { '-g' { res.is_debug = true res.is_vlines = true + res.build_options << arg } '-cg' { res.is_debug = true res.is_vlines = false + res.build_options << arg } '-repl' { res.is_repl = true @@ -190,19 +195,23 @@ pub fn parse_args(args []string) (&Preferences, string) { } '-autofree' { res.autofree = true + res.build_options << arg } '-compress' { res.compress = true } '-freestanding' { res.is_bare = true + res.build_options << arg } '-no-preludes' { res.no_preludes = true + res.build_options << arg } '-prof', '-profile' { res.profile_file = cmdline.option(current_args, '-profile', '-') res.is_prof = true + res.build_options << '$arg $res.profile_file' i++ } '-profile-no-inline' { @@ -210,6 +219,7 @@ pub fn parse_args(args []string) (&Preferences, string) { } '-prod' { res.is_prod = true + res.build_options << arg } '-simulator' { res.is_ios_simulator = true @@ -240,6 +250,7 @@ pub fn parse_args(args []string) (&Preferences, string) { } '-prealloc' { res.prealloc = true + res.build_options << arg } '-keepc' { eprintln('-keepc is deprecated. V always keeps the generated .tmp.c files now.') @@ -249,6 +260,7 @@ pub fn parse_args(args []string) (&Preferences, string) { } '-x64' { res.backend = .x64 + res.build_options << arg } '-W' { res.warns_are_errors = true @@ -277,6 +289,7 @@ pub fn parse_args(args []string) (&Preferences, string) { exit(1) } res.os = target_os_kind + res.build_options << '$arg $target_os' } '-printfn' { res.printfn_list << cmdline.option(current_args, '-printfn', '') @@ -284,6 +297,7 @@ pub fn parse_args(args []string) (&Preferences, string) { } '-cflags' { res.cflags += ' ' + cmdline.option(current_args, '-cflags', '') + res.build_options << '$arg "$res.cflags.trim_space()"' i++ } '-define', '-d' { @@ -295,6 +309,7 @@ pub fn parse_args(args []string) (&Preferences, string) { } '-cc' { res.ccompiler = cmdline.option(current_args, '-cc', 'cc') + res.build_options << '$arg "$res.ccompiler"' i++ } '-o' { @@ -302,7 +317,9 @@ pub fn parse_args(args []string) (&Preferences, string) { i++ } '-b' { - b := backend_from_string(cmdline.option(current_args, '-b', 'c')) or { + sbackend := cmdline.option(current_args, '-b', 'c') + res.build_options << '$arg $sbackend' + b := backend_from_string(sbackend) or { continue } res.backend = b @@ -310,11 +327,13 @@ pub fn parse_args(args []string) (&Preferences, string) { } '-path' { path := cmdline.option(current_args, '-path', '') + res.build_options << '$arg "$path"' res.lookup_path = path.replace('|', os.path_delimiter).split(os.path_delimiter) i++ } '-custom-prelude' { path := cmdline.option(current_args, '-custom-prelude', '') + res.build_options << '$arg $path' prelude := os.read_file(path) or { eprintln('cannot open custom prelude file: $err') exit(1) @@ -390,6 +409,13 @@ pub fn parse_args(args []string) (&Preferences, string) { res.build_mode = .build_module res.path = args[command_pos + 1] } + // keep only the unique res.build_options: + mut m := map[string]string{} + for x in res.build_options { + m[x] = '' + } + res.build_options = m.keys() + // eprintln('>> res.build_options: $res.build_options') res.fill_with_defaults() return res, command } diff --git a/vlib/v/pref/vcache.v b/vlib/v/pref/vcache.v new file mode 100644 index 0000000000..ac9f689007 --- /dev/null +++ b/vlib/v/pref/vcache.v @@ -0,0 +1,85 @@ +module pref + +import os +import crypto.md5 + +// Using a 2 level cache, ensures a more even distribution of cache entries, +// so there will not be cramped folders that contain many thousands of them. +// Most filesystems can not handle performantly such folders, and slow down. +// The first level will contain a max of 256 folders, named from 00/ to ff/. +// Each of them will contain many entries, but hopefully < 1000. +// NB: using a hash here, makes the cache storage immune to special +// characters in the keys, like quotes, spaces and so on. +// Cleanup of the cache is simple: just delete the $VCACHE folder. +// The cache tree will look like this: +// │ $VCACHE +// │ ├── 0f +// │ │ ├── 0f004f983ab9c487b0d7c1a0a73840a5.txt +// │ │ ├── 0f599edf5e16c2756fbcdd4c865087ac.description.txt <-- build details +// │ │ └── 0f599edf5e16c2756fbcdd4c865087ac.vh +// │ ├── 29 +// │ │ ├── 294717dd02a1cca5f2a0393fca2c5c22.o +// │ │ └── 294717dd02a1cca5f2a0393fca2c5c22.description.txt <-- build details +// │ ├── 62 +// │ │ └── 620d60d6b81fdcb3cab030a37fd86996.h +// │ └── 76 +// │ └── 7674f983ab9c487b0d7c1a0ad73840a5.c +pub struct CacheManager { +pub: + basepath string +pub mut: + vopts string + k2cpath map[string]string // key -> filesystem cache path for the object +} + +fn new_cache_manager(opts []string) CacheManager { + mut vcache_basepath := os.getenv('VCACHE') + if vcache_basepath == '' { + vcache_basepath = os.join_path(os.home_dir(), '.vmodules', 'cache') + } + return CacheManager{ + basepath: vcache_basepath + vopts: opts.join('|') + } +} + +pub fn (mut cm CacheManager) key2cpath(key string) string { + mut cpath := cm.k2cpath[key] + if cpath == '' { + hk := cm.vopts + key + hash := md5.sum(hk.bytes()).hex() + prefix := hash[0..2] + cprefix_folder := os.join_path(cm.basepath, prefix) + cpath = os.join_path(cprefix_folder, hash) + if !os.is_dir(cprefix_folder) { + os.mkdir_all(cprefix_folder) + os.chmod(cprefix_folder, 0o777) + } + cm.k2cpath[key] = cpath + } + return cpath +} + +pub fn (mut cm CacheManager) postfix_with_key2cpath(postfix string, key string) string { + return cm.key2cpath(key) + postfix +} + +pub fn (mut cm CacheManager) exists(postfix string, key string) ?string { + fpath := cm.postfix_with_key2cpath(postfix, key) + if !os.exists(fpath) { + return error('does not exist yet') + } + return fpath +} + +pub fn (mut cm CacheManager) save(postfix string, key string, content string) ?string { + fpath := cm.postfix_with_key2cpath(postfix, key) + os.write_file(fpath, content) ? + return fpath +} + +pub fn (mut cm CacheManager) load(postfix string, key string) ?string { + fpath := cm.exists(postfix, key) ? + content := os.read_file(fpath) ? + return content +}