vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

v: add VCACHE support for thirdparty object files and for `v build-module`

pull/6678/head
Delyan Angelov 2020-10-25 03:09:07 +03:00
parent 89daec4e93
commit 5f6259dde6
6 changed files with 166 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
vlib/v/builder/cc.v
View File

@ -118,6 +118,21 @@ fn (mut v Builder) post_process_c_compiler_output(res os.Result) {
verror(c_error_info)
}
fn (mut v Builder) rebuild_cached_module(vexe string, imp_path string) string {
res := v.pref.cache_manager.exists('.o', imp_path) or {
println('Cached $imp_path .o file not found... Building .o file for $imp_path')
boptions := v.pref.build_options.join(' ')
rebuild_cmd := '$vexe $boptions build-module $imp_path'
// eprintln('>> rebuild_cmd: $rebuild_cmd')
os.system(rebuild_cmd)
rebuilded_o := v.pref.cache_manager.exists('.o', imp_path) or {
panic('could not rebuild cache module for $imp_path, error: $err')
}
return rebuilded_o
}
return res
}
fn (mut v Builder) cc() {
if os.executable().contains('vfmt') {
return
@ -260,14 +275,9 @@ fn (mut v Builder) cc() {
linker_flags << '-nostdlib'
}
if v.pref.build_mode == .build_module {
// Create the modules & out directory if it's not there.
out_dir := os.join_path(pref.default_module_path, 'cache', v.pref.path)
pdir := out_dir.all_before_last(os.path_separator)
if !os.is_dir(pdir) {
os.mkdir_all(pdir)
}
v.pref.out_name = '${out_dir}.o' // v.out_name
println('Building ${v.pref.out_name}...')
v.pref.out_name = v.pref.cache_manager.postfix_with_key2cpath('.o', v.pref.path) // v.out_name
println('Building $v.pref.path to $v.pref.out_name ...')
v.pref.cache_manager.save('.description.txt', v.pref.path, '${v.pref.path:-30} @ $v.pref.cache_manager.vopts\n')
// println('v.table.imports:')
// println(v.table.imports)
}
@ -349,10 +359,7 @@ fn (mut v Builder) cc() {
args << '-c'
} else if v.pref.use_cache {
mut built_modules := []string{}
builtin_obj_path := os.join_path(pref.default_module_path, 'cache', 'vlib', 'builtin.o')
if !os.exists(builtin_obj_path) {
os.system('$vexe build-module vlib/builtin')
}
builtin_obj_path := v.rebuild_cached_module(vexe, 'vlib/builtin')
libs += ' ' + builtin_obj_path
for ast_file in v.parsed_files {
for imp_stmt in ast_file.imports {
@ -382,14 +389,8 @@ fn (mut v Builder) cc() {
// continue
// }
imp_path := os.join_path('vlib', mod_path)
cache_path := os.join_path(pref.default_module_path, 'cache')
obj_path := os.join_path(cache_path, '${imp_path}.o')
if os.exists(obj_path) {
libs += ' ' + obj_path
} else {
println('$obj_path not found... building module $imp')
os.system('$vexe build-module $imp_path')
}
obj_path := v.rebuild_cached_module(vexe, imp_path)
libs += ' ' + obj_path
if obj_path.ends_with('vlib/ui.o') {
args << '-framework Cocoa -framework Carbon'
}
@ -777,21 +778,19 @@ fn (mut v Builder) build_thirdparty_obj_file(path string, moduleflags []cflag.CF
if v.pref.os == .windows {
// Cross compiling for Windows
$if !windows {
if os.exists(obj_path) {
os.rm(obj_path)
}
v.pref.ccompiler = mingw_cc
}
}
if os.exists(obj_path) {
opath := v.pref.cache_manager.postfix_with_key2cpath('.o', obj_path)
if os.exists(opath) {
return
}
println('$obj_path not found, building it...')
println('$obj_path not found, building it in $opath ...')
cfile := '${path[..path.len - 2]}.c'
btarget := moduleflags.c_options_before_target()
atarget := moduleflags.c_options_after_target()
cppoptions := if v.pref.ccompiler.contains('++') { ' -fpermissive -w ' } else { '' }
cmd := '$v.pref.ccompiler $cppoptions $v.pref.third_party_option $btarget -c -o "$obj_path" "$cfile" $atarget'
cmd := '$v.pref.ccompiler $cppoptions $v.pref.third_party_option $btarget -c -o "$opath" "$cfile" $atarget'
res := os.exec(cmd) or {
eprintln('exec failed for thirdparty object build cmd:\n$cmd')
verror(err)
@ -802,6 +801,7 @@ fn (mut v Builder) build_thirdparty_obj_file(path string, moduleflags []cflag.CF
verror(res.output)
return
}
v.pref.cache_manager.save('.description.txt', obj_path, '${obj_path:-30} @ $cmd\n')
println(res.output)
}

10
vlib/v/builder/cflags.v
View File

@ -1,15 +1,19 @@
module builder
import os
import v.cflag
// get flags for current os
fn (v &Builder) get_os_cflags() []cflag.CFlag {
fn (mut v Builder) get_os_cflags() []cflag.CFlag {
mut flags := []cflag.CFlag{}
mut ctimedefines := []string{}
if v.pref.compile_defines.len > 0 {
ctimedefines << v.pref.compile_defines
}
for flag in v.table.cflags {
for mut flag in v.table.cflags {
if flag.value.ends_with('.o') {
flag.cached = v.pref.cache_manager.postfix_with_key2cpath('.o', os.real_path(flag.value))
}
if flag.os == '' ||
(flag.os == 'linux' && v.pref.os == .linux) ||
(flag.os == 'macos' && v.pref.os == .macos) ||
@ -27,7 +31,7 @@ fn (v &Builder) get_os_cflags() []cflag.CFlag {
return flags
}
fn (v &Builder) get_rest_of_module_cflags(c &cflag.CFlag) []cflag.CFlag {
fn (mut v Builder) get_rest_of_module_cflags(c &cflag.CFlag) []cflag.CFlag {
mut flags := []cflag.CFlag{}
cflags := v.get_os_cflags()
for flag in cflags {

15
vlib/v/cflag/cflags.v
View File

@ -8,19 +8,24 @@ import os
// parsed cflag
pub struct CFlag {
pub:
mod string // the module in which the flag was given
os string // eg. windows | darwin | linux
name string // eg. -I
value string // eg. /path/to/include
mod string // the module in which the flag was given
os string // eg. windows | darwin | linux
name string // eg. -I
value string // eg. /path/to/include
pub mut:
cached string // eg. ~/.vmodules/cache/ea/ea9878886727367672163.o (for .o files)
}
pub fn (c &CFlag) str() string {
return 'CFlag{ name: "$c.name" value: "$c.value" mod: "$c.mod" os: "$c.os" }'
return 'CFlag{ name: "$c.name" value: "$c.value" mod: "$c.mod" os: "$c.os" cached: "$c.cached" }'
}
// format flag
pub fn (cf &CFlag) format() string {
mut value := cf.value
if cf.cached != '' {
value = cf.cached
}
if cf.name in ['-l', '-Wa', '-Wl', '-Wp'] && value.len > 0 {
return '$cf.name$value'.trim_space()
}

11
vlib/v/pref/default.v
View File

@ -71,6 +71,17 @@ pub fn (mut p Preferences) fill_with_defaults() {
}
}
}
// Prepare the cache manager. All options that can affect the generated cached .c files
// should go into res.cache_manager.vopts, which is used as a salt for the cache hash.
p.cache_manager = new_cache_manager([
'$p.backend | $p.os | $p.ccompiler',
p.cflags.trim_space(),
p.third_party_option.trim_space(),
'$p.compile_defines_all',
'$p.compile_defines',
'$p.lookup_path',
])
// eprintln('prefs.cache_manager: $p')
}
fn default_c_compiler() string {

28
vlib/v/pref/pref.v
View File

@ -129,6 +129,8 @@ pub mut:
is_ios_simulator bool
is_apk bool // build as Android .apk format
cleanup_files []string // list of temporary *.tmp.c and *.tmp.c.rsp files. Cleaned up on successfull builds.
build_options []string // list of options, that should be passed down to `build-module`, if needed for -usecache
cache_manager CacheManager
}
pub fn parse_args(args []string) (&Preferences, string) {
@ -142,6 +144,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
match arg {
'-apk' {
res.is_apk = true
res.build_options << arg
}
'-show-timings' {
res.show_timings = true
@ -167,10 +170,12 @@ pub fn parse_args(args []string) (&Preferences, string) {
'-g' {
res.is_debug = true
res.is_vlines = true
res.build_options << arg
}
'-cg' {
res.is_debug = true
res.is_vlines = false
res.build_options << arg
}
'-repl' {
res.is_repl = true
@ -190,19 +195,23 @@ pub fn parse_args(args []string) (&Preferences, string) {
}
'-autofree' {
res.autofree = true
res.build_options << arg
}
'-compress' {
res.compress = true
}
'-freestanding' {
res.is_bare = true
res.build_options << arg
}
'-no-preludes' {
res.no_preludes = true
res.build_options << arg
}
'-prof', '-profile' {
res.profile_file = cmdline.option(current_args, '-profile', '-')
res.is_prof = true
res.build_options << '$arg $res.profile_file'
i++
}
'-profile-no-inline' {
@ -210,6 +219,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
}
'-prod' {
res.is_prod = true
res.build_options << arg
}
'-simulator' {
res.is_ios_simulator = true
@ -240,6 +250,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
}
'-prealloc' {
res.prealloc = true
res.build_options << arg
}
'-keepc' {
eprintln('-keepc is deprecated. V always keeps the generated .tmp.c files now.')
@ -249,6 +260,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
}
'-x64' {
res.backend = .x64
res.build_options << arg
}
'-W' {
res.warns_are_errors = true
@ -277,6 +289,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
exit(1)
}
res.os = target_os_kind
res.build_options << '$arg $target_os'
}
'-printfn' {
res.printfn_list << cmdline.option(current_args, '-printfn', '')
@ -284,6 +297,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
}
'-cflags' {
res.cflags += ' ' + cmdline.option(current_args, '-cflags', '')
res.build_options << '$arg "$res.cflags.trim_space()"'
i++
}
'-define', '-d' {
@ -295,6 +309,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
}
'-cc' {
res.ccompiler = cmdline.option(current_args, '-cc', 'cc')
res.build_options << '$arg "$res.ccompiler"'
i++
}
'-o' {
@ -302,7 +317,9 @@ pub fn parse_args(args []string) (&Preferences, string) {
i++
}
'-b' {
b := backend_from_string(cmdline.option(current_args, '-b', 'c')) or {
sbackend := cmdline.option(current_args, '-b', 'c')
res.build_options << '$arg $sbackend'
b := backend_from_string(sbackend) or {
continue
}
res.backend = b
@ -310,11 +327,13 @@ pub fn parse_args(args []string) (&Preferences, string) {
}
'-path' {
path := cmdline.option(current_args, '-path', '')
res.build_options << '$arg "$path"'
res.lookup_path = path.replace('|', os.path_delimiter).split(os.path_delimiter)
i++
}
'-custom-prelude' {
path := cmdline.option(current_args, '-custom-prelude', '')
res.build_options << '$arg $path'
prelude := os.read_file(path) or {
eprintln('cannot open custom prelude file: $err')
exit(1)
@ -390,6 +409,13 @@ pub fn parse_args(args []string) (&Preferences, string) {
res.build_mode = .build_module
res.path = args[command_pos + 1]
}
// keep only the unique res.build_options:
mut m := map[string]string{}
for x in res.build_options {
m[x] = ''
}
res.build_options = m.keys()
// eprintln('>> res.build_options: $res.build_options')
res.fill_with_defaults()
return res, command
}

85
vlib/v/pref/vcache.v 100644
View File

@ -0,0 +1,85 @@
module pref
import os
import crypto.md5
// Using a 2 level cache, ensures a more even distribution of cache entries,
// so there will not be cramped folders that contain many thousands of them.
// Most filesystems can not handle performantly such folders, and slow down.
// The first level will contain a max of 256 folders, named from 00/ to ff/.
// Each of them will contain many entries, but hopefully < 1000.
// NB: using a hash here, makes the cache storage immune to special
// characters in the keys, like quotes, spaces and so on.
// Cleanup of the cache is simple: just delete the $VCACHE folder.
// The cache tree will look like this:
// │ $VCACHE
// │ ├── 0f
// │ │ ├── 0f004f983ab9c487b0d7c1a0a73840a5.txt
// │ │ ├── 0f599edf5e16c2756fbcdd4c865087ac.description.txt <-- build details
// │ │ └── 0f599edf5e16c2756fbcdd4c865087ac.vh
// │ ├── 29
// │ │ ├── 294717dd02a1cca5f2a0393fca2c5c22.o
// │ │ └── 294717dd02a1cca5f2a0393fca2c5c22.description.txt <-- build details
// │ ├── 62
// │ │ └── 620d60d6b81fdcb3cab030a37fd86996.h
// │ └── 76
// │ └── 7674f983ab9c487b0d7c1a0ad73840a5.c
pub struct CacheManager {
pub:
basepath string
pub mut:
vopts string
k2cpath map[string]string // key -> filesystem cache path for the object
}
fn new_cache_manager(opts []string) CacheManager {
mut vcache_basepath := os.getenv('VCACHE')
if vcache_basepath == '' {
vcache_basepath = os.join_path(os.home_dir(), '.vmodules', 'cache')
}
return CacheManager{
basepath: vcache_basepath
vopts: opts.join('|')
}
}
pub fn (mut cm CacheManager) key2cpath(key string) string {
mut cpath := cm.k2cpath[key]
if cpath == '' {
hk := cm.vopts + key
hash := md5.sum(hk.bytes()).hex()
prefix := hash[0..2]
cprefix_folder := os.join_path(cm.basepath, prefix)
cpath = os.join_path(cprefix_folder, hash)
if !os.is_dir(cprefix_folder) {
os.mkdir_all(cprefix_folder)
os.chmod(cprefix_folder, 0o777)
}
cm.k2cpath[key] = cpath
}
return cpath
}
pub fn (mut cm CacheManager) postfix_with_key2cpath(postfix string, key string) string {
return cm.key2cpath(key) + postfix
}
pub fn (mut cm CacheManager) exists(postfix string, key string) ?string {
fpath := cm.postfix_with_key2cpath(postfix, key)
if !os.exists(fpath) {
return error('does not exist yet')
}
return fpath
}
pub fn (mut cm CacheManager) save(postfix string, key string, content string) ?string {
fpath := cm.postfix_with_key2cpath(postfix, key)
os.write_file(fpath, content) ?
return fpath
}
pub fn (mut cm CacheManager) load(postfix string, key string) ?string {
fpath := cm.exists(postfix, key) ?
content := os.read_file(fpath) ?
return content
}