vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

compiler: cache modules

pull/2229/head
Alexander Medvednikov 2019-10-04 15:48:09 +03:00 committed by GitHub
parent f45d3f07ed
commit 52f4f4026b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 422 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
compiler/cc.v
View File

@ -63,11 +63,12 @@ fn (v mut V) cc() {
v.out_name = v.out_name + '.so'
}
if v.pref.build_mode == .build_module {
// Create the modules directory if it's not there.
if !os.file_exists(v_modules_path) {
os.mkdir(v_modules_path)
// Create the modules & out directory if it's not there.
out_dir := v_modules_path + v.dir
if !os.dir_exists(out_dir) {
os.mkdir(out_dir)
}
v.out_name = v_modules_path + v.dir + '.o' //v.out_name
v.out_name = '${out_dir}.o' //v.out_name
println('Building ${v.out_name}...')
}

4
compiler/comptime.v
View File

@ -117,7 +117,7 @@ fn (p mut Parser) comp_time() {
// Parse the function and embed resulting C code in current function so that
// all variables are available.
pos := p.cgen.lines.len - 1
mut pp := p.v.new_parser_file('.vwebtmpl.v')
mut pp := p.v.new_parser_from_file('.vwebtmpl.v')
if !p.pref.is_debug {
os.rm('.vwebtmpl.v')
}
@ -279,7 +279,7 @@ fn (p mut Parser) gen_struct_str(typ Type) {
for field in typ.fields {
sb.writeln('\t$field.name: $' + 'a.${field.name}')
}
sb.writeln("\n}'")
sb.writeln("}'")
sb.writeln('}')
p.v.vgen_buf.writeln(sb.str())
// Need to manually add the definition to `fns` so that it stays

17
compiler/fn.v
View File

@ -144,6 +144,7 @@ fn (p mut Parser) is_sig() bool {
fn (p mut Parser) fn_decl() {
p.clear_vars() // clear local vars every time a new fn is started
p.fgen('fn ')
//defer { p.fgenln('\n') }
// If we are in the first pass, create a new function.
// In the second pass fetch the one we created.
@ -426,10 +427,17 @@ fn (p mut Parser) fn_decl() {
if !is_c && p.first_pass() {
// TODO hack to make Volt compile without -embed_vlib
if f.name == 'darwin__nsstring' && p.pref.build_mode == .default_mode {
return
}
} else {
p.cgen.fns << fn_decl + ';'
}
}
// Generate .vh header files when building a module
/*
if p.pref.build_mode == .build_module {
p.vh_genln(f.v_definition())
}
*/
return
}
if p.attr == 'live' && p.pref.is_so {
@ -1092,11 +1100,6 @@ fn (f &Fn) typ_str() string {
return sb.str()
}
// "fn foo(a int) stirng", for .vh module headers
fn (f &Fn) v_definition() string {
return 'fn '//$f.name(${f.str_args()})'
}
// f.args => "int a, string b"
fn (f &Fn) str_args(table &Table) string {
mut s := ''

253
compiler/main.v
View File

@ -29,7 +29,6 @@ enum BuildMode {
const (
supported_platforms = ['windows', 'mac', 'linux', 'freebsd', 'openbsd',
'netbsd', 'dragonfly', 'msvc', 'android', 'js', 'solaris']
v_modules_path = os.home_dir() + '/.vmodules/'
)
enum OS {
@ -105,11 +104,11 @@ mut:
building_v bool
autofree bool
compress bool
// Skips re-compilation of the builtin module
skip_builtin bool // Skips re-compilation of the builtin module
// to increase compilation time.
// This is on by default, since a vast majority of users do not
// work on the builtin module itself.
skip_builtin bool
}
fn main() {
@ -202,9 +201,7 @@ fn main() {
}
v.table.fns.free()
free(v.table)
//for p in parsers {
//}
//for p in parsers {}
println('done!')
}
}
@ -218,13 +215,22 @@ fn (v mut V) add_parser(parser Parser) {
v.parsers << parser
}
fn (v mut V) parse(file string, pass Pass) {
//println('parse($file, $pass)')
for i, p in v.parsers {
if p.file_path == file {
v.parsers[i].parse(pass)
return
}
}
}
fn (v mut V) compile() {
// Emily: Stop people on linux from being able to build with msvc
if os.user_os() != 'windows' && v.os == .msvc {
verror('Cannot build with msvc on ${os.user_os()}')
}
mut cgen := v.cgen
cgen.genln('// Generated by V')
if v.pref.is_verbose {
@ -250,12 +256,7 @@ fn (v mut V) compile() {
*/
// First pass (declarations)
for file in v.files {
for i, p in v.parsers {
if p.file_path == file {
v.parsers[i].parse(.decl)
break
}
}
v.parse(file, .decl)
}
// Main pass
cgen.pass = Pass.main
@ -321,12 +322,7 @@ fn (v mut V) compile() {
cgen.genln('this line will be replaced with definitions')
defs_pos := cgen.lines.len - 1
for file in v.files {
for i, p in v.parsers {
if p.file_path == file {
v.parsers[i].parse(.main)
break
}
}
v.parse(file, .main)
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
// p.g.gen_x64()
// Format all files (don't format automatically generated vlib headers)
@ -334,8 +330,12 @@ fn (v mut V) compile() {
// new vfmt is not ready yet
}
}
// Generate .vh if we are building a module
if v.pref.build_mode == .build_module {
v.generate_vh()
}
// parse generated V code (str() methods etc)
mut vgen_parser := v.new_parser_string(v.vgen_buf.str(), 'vgen')
mut vgen_parser := v.new_parser_from_string(v.vgen_buf.str(), 'vgen')
// free the string builder which held the generated methods
v.vgen_buf.free()
vgen_parser.parse(.main)
@ -573,8 +573,63 @@ fn (v &V) v_files_from_dir(dir string) []string {
// Parses imports, adds necessary libs, and then user files
fn (v mut V) add_v_files_to_compile() {
// Parse builtin imports
for file in v.get_builtin_files() {
// add builtins first
v.files << file
mut p := v.new_parser_from_file(file)
p.parse(.imports)
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
}
// Parse user imports
for file in v.get_user_files() {
mut p := v.new_parser_from_file(file)
p.parse(.imports)
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
}
// Parse lib imports
v.parse_lib_imports()
if v.pref.is_verbose {
v.log('imports:')
println(v.table.imports)
}
// resolve deps & add imports in correct order
for mod in v.resolve_deps().imports() {
// if mod == v.mod { continue } // Building this module? Skip. TODO it's a hack.
if mod == 'main' { continue } // main files will get added last
// use cached built module if exists
vh_path := '$v_modules_path/${mod}.vh'
if os.file_exists(vh_path) {
println('using cached module `$mod`: $vh_path')
v.files << vh_path
continue
}
// standard module
vfiles := v.v_files_from_dir(v.find_module_path(mod))
for file in vfiles {
v.files << file
}
}
// add remaining main files last
for _, fit in v.table.file_imports {
if fit.module_name != 'main' { continue }
v.files << fit.file_path
}
}
// get builtin files
fn (v &V) get_builtin_files() []string {
$if js {
return v.v_files_from_dir('$v.vroot/vlib/builtin/js/')
}
return v.v_files_from_dir('$v.vroot/vlib/builtin/')
}
// get user files
fn (v &V) get_user_files() []string {
mut dir := v.dir
v.log('add_v_files($dir)')
v.log('get_v_files($dir)')
// Need to store user files separately, because they have to be added after libs, but we dont know
// which libs need to be added yet
mut user_files := []string
@ -607,46 +662,25 @@ fn (v mut V) add_v_files_to_compile() {
v.log('user_files:')
println(user_files)
}
// Parse builtin imports
for file in v.files {
mut p := v.new_parser_file(file)
p.parse(.imports)
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
}
// Parse user imports
for file in user_files {
mut p := v.new_parser_file(file)
p.parse(.imports)
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
}
// Parse lib imports
/*
if v.pref.build_mode == .default_mode {
// strange ( for mod in v.table.imports ) dosent loop all items
// for mod in v.table.imports {
for i := 0; i < v.table.imports.len; i++ {
mod := v.table.imports[i]
mod_path := v.module_path(mod)
import_path := '$v_modules_path/vlib/$mod_path'
vfiles := v.v_files_from_dir(import_path)
if vfiles.len == 0 {
verror('cannot import module $mod (no .v files in "$import_path")')
}
// Add all imports referenced by these libs
for file in vfiles {
mut p := v.new_parser_file(file, Pass.imports)
p.parse()
return user_files
}
if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
// parse deps from already parsed builtin/user files
fn (v mut V) parse_lib_imports() {
mut done_fits := []string
for {
for _, fit in v.table.file_imports {
if fit.file_path in done_fits { continue }
v.parse_file_imports(fit)
done_fits << fit.file_path
}
if v.table.file_imports.size == done_fits.len { break}
}
}
else {
*/
// strange ( for mod in v.table.imports ) dosent loop all items
// for mod in v.table.imports {
for i := 0; i < v.table.imports.len; i++ {
mod := v.table.imports[i]
}
// parse imports from file import table
fn (v mut V) parse_file_imports(fit &FileImportTable) {
for _, mod in fit.imports {
import_path := v.find_module_path(mod)
vfiles := v.v_files_from_dir(import_path)
if vfiles.len == 0 {
@ -654,7 +688,7 @@ fn (v mut V) add_v_files_to_compile() {
}
// Add all imports referenced by these libs
for file in vfiles {
mut p := v.new_parser_file(file)
mut p := v.new_parser_from_file(file)
p.parse(.imports)
if p.import_table.module_name != mod {
verror('bad module name: $file was imported as `$mod` but it is defined as module `$p.import_table.module_name`')
@ -662,65 +696,18 @@ fn (v mut V) add_v_files_to_compile() {
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
}
}
if v.pref.is_verbose {
v.log('imports:')
println(v.table.imports)
}
// graph deps
}
// return resolved dep graph (order deps)
fn (v &V) resolve_deps() &DepGraph {
mut dep_graph := new_dep_graph()
dep_graph.from_import_tables(v.table.file_imports)
deps_resolved := dep_graph.resolve()
if !deps_resolved.acyclic {
deps_resolved.display()
verror('Import cycle detected')
}
// add imports in correct order
for mod in deps_resolved.imports() {
// Building this module? Skip. TODO it's a hack.
if mod == v.mod {
continue
}
mod_path := v.find_module_path(mod)
// If we are in default mode, we don't parse vlib .v files, but
// header .vh files in
// TmpPath/vlib
// These were generated by vfmt
/*
if v.pref.build_mode == .default_mode || v.pref.build_mode == .build_module {
module_path = '$v_modules_path/vlib/$mod_p'
}
*/
if mod == 'builtin' { continue } // builtin files were already added
vfiles := v.v_files_from_dir(mod_path)
for file in vfiles {
if !(file in v.files) {
v.files << file
}
}
}
// Add remaining user files
mut i := 0
mut j := 0
mut len := -1
for _, fit in v.table.file_imports {
// Don't add a duplicate; builtin files are always there
if fit.file_path in v.files || fit.module_name == 'builtin' {
i++
continue
}
if len == -1 {
len = i
}
j++
// TODO remove this once imports work with .build
if v.pref.build_mode == .build_module && j >= len / 2{
break
}
//println(fit)
//println('fit $fit.file_path')
v.files << fit.file_path
i++
verror('import cycle detected')
}
return deps_resolved
}
fn get_arg(joined_args, arg, def string) string {
@ -743,15 +730,6 @@ fn get_all_after(joined_args, arg, def string) string {
return res
}
fn (v &V) module_path(mod string) string {
// submodule support
if mod.contains('.') {
//return mod.replace('.', os.PathSeparator)
return mod.replace('.', '/')
}
return mod
}
fn (v &V) log(s string) {
if !v.pref.is_verbose {
return
@ -795,14 +773,14 @@ fn new_v(args[]string) &V {
//out_name = '$TmpPath/vlib/${base}.o'
out_name = mod + '.o'
// Cross compiling? Use separate dirs for each os
/*
/*
if target_os != os.user_os() {
os.mkdir('$TmpPath/vlib/$target_os')
out_name = '$TmpPath/vlib/$target_os/${base}.o'
println('target_os=$target_os user_os=${os.user_os()}')
println('!Cross compiling $out_name')
}
*/
*/
}
// TODO embed_vlib is temporarily the default mode. It's much slower.
else if !('-embed_vlib' in args) {
@ -867,18 +845,7 @@ fn new_v(args[]string) &V {
}
}
//println('OS=$_os')
builtin := 'builtin.v'
builtins := [
'array.v',
'string.v',
'builtin.v',
'int.v',
'utf8.v',
'map.v',
'hashmap.v',
'option.v',
]
//println(builtins)
// Location of all vlib files
vroot := os.dir(os.executable())
//println('VROOT=$vroot')
@ -898,21 +865,6 @@ fn new_v(args[]string) &V {
}
//println('out_name:$out_name')
mut out_name_c := os.realpath( out_name ) + '.tmp.c'
mut files := []string
// Add builtin files
//if !out_name.contains('builtin.o') {
for builtin in builtins {
mut f := '$vroot/vlib/builtin/$builtin'
__ := 1
$if js {
f = '$vroot/vlib/builtin/js/$builtin'
}
// In default mode we use precompiled vlib.o, point to .vh files with signatures
if build_mode == .default_mode || build_mode == .build_module {
//f = '$TmpPath/vlib/builtin/${builtin}h'
}
files << f
}
cflags := get_cmdline_cflags(args)
@ -954,7 +906,6 @@ fn new_v(args[]string) &V {
return &V{
os: _os
out_name: out_name
files: files
dir: dir
lang_dir: vroot
table: new_table(obfuscate)

201
compiler/module_header.v 100644
View File

@ -0,0 +1,201 @@
// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module main
import (
strings
os
)
/* .vh generation logic.
.vh files contains only function signatures, consts, and types.
They are used together with pre-compiled modules.
*/
// "fn foo(a int) string"
fn (f &Fn) v_definition() string {
//t :=time.ticks()
mut sb := strings.new_builder(100)
if f.is_public {
sb.write('pub ')
}
sb.write('fn ')
if f.is_c {
sb.write('C.')
}
if f.is_method {
recv := f.args[0]
typ := v_type_str(recv.typ)
mut mu := if recv.is_mut { 'mut' } else { '' }
if recv.ref {
mu = '&'
}
sb.write('($recv.name $mu $typ) ')
}
if f.name.contains('__') {
sb.write(f.name.all_after('__') + '(')
} else {
sb.write('$f.name(')
}
for i, arg in f.args {
typ := v_type_str(arg.typ)
if arg.name == '' {
sb.write(typ)
} else {
sb.write('$arg.name $typ')
}
if i != f.args.len - 1 {
sb.write(', ')
}
}
sb.write(')')
if f.typ != 'void' {
typ := v_type_str(f.typ)
sb.write(' ')
sb.write(typ)
sb.writeln(' ')
}
//println('ms: ${time.ticks() - t}')
return sb.str()
}
fn v_type_str(typ_ string) string {
typ := if typ_.ends_with('*') {
'*' + typ_.left(typ_.len - 1)
} else {
typ_
}
//println('"$typ"')
if typ == '*void' {
return 'voidptr'
}
if typ == '*byte' {
return 'byteptr'
}
if typ.starts_with('array_') {
return '[]' + typ.right(6)
}
if typ.contains('__') {
return typ.all_after('__')
}
return typ.replace('Option_', '?')
}
fn (v &V) generate_vh() {
println('Generating a V header file for module `$v.mod`')
dir := v_modules_path + v.mod
path := dir + '.vh'
if !os.dir_exists(dir) {
os.mkdir(dir)
}
println(path)
file := os.create(path) or { panic(err) }
// Consts
file.writeln('// $v.mod module header \n')
file.writeln('// Consts')
if v.table.consts.len > 0 {
file.writeln('const (')
for i, c in v.table.consts {
if c.mod != v.mod {
continue
}
println('$i $c.name')
//if !c.name.contains('__') {
//continue
//}
name := c.name.all_after('__')
typ := v_type_str(c.typ)
file.writeln('\t$name $typ')
}
file.writeln(')\n')
// Globals
for var in v.table.consts {
if var.mod != v.mod {
continue
}
if !var.is_global {
continue
}
name := var.name.all_after('__')
typ := v_type_str(var.typ)
file.writeln('__global $name $typ')
}
file.writeln('\n')
}
// Types
file.writeln('// Types')
for _, typ in v.table.typesmap {
if typ.mod != v.mod {
continue
}
if typ.name.contains('__') {
continue
}
if typ.cat == .struct_ {
file.writeln('struct $typ.name {')
// Private fields
for field in typ.fields {
if field.access_mod == .public {
continue
}
field_type := v_type_str(field.typ)
file.writeln('\t$field.name $field_type')
}
file.writeln('pub:')
for field in typ.fields {
if field.access_mod == .private {
continue
}
field_type := v_type_str(field.typ)
file.writeln('\t$field.name $field_type')
}
file.writeln('}\n')
}
}
// Functions & methods
file.writeln('// Functions')
// Public first
mut fns := []Fn
// TODO fns := v.table.fns.filter(.mod == v.mod)
for _, f in v.table.fns {
if f.mod == v.mod {
fns << f
}
}
for _, f in fns {
if !f.is_public {
continue
}
file.writeln(f.v_definition())
}
// Private
for _, f in fns {
if f.is_public {
continue
}
file.writeln(f.v_definition())
}
// Methods
file.writeln('\n// Methods //////////////////')
for _, typ in v.table.typesmap {
if typ.mod != v.mod {
continue
}
for method in typ.methods {
file.writeln(method.v_definition())
}
}
file.close()
/*
for i, p in v.parsers {
if v.parsers[i].vh_lines.len > 0 {
os.write_file(p.file_name +'.vh', v.parsers[i].vh_lines.join('\n'))
}
}
*/
}

16
compiler/modules.v
View File

@ -6,6 +6,10 @@ module main
import os
const (
v_modules_path = os.home_dir() + '/.vmodules/'
)
// add a module and its deps (module speficic dag method)
pub fn(graph mut DepGraph) from_import_tables(import_tables map[string]FileImportTable) {
for _, fit in import_tables {
@ -21,14 +25,20 @@ pub fn(graph mut DepGraph) from_import_tables(import_tables map[string]FileImpor
pub fn(graph &DepGraph) imports() []string {
mut mods := []string
for node in graph.nodes {
if node.name == 'main' {
continue
}
mods << node.name
}
return mods
}
fn (v &V) module_path(mod string) string {
// submodule support
if mod.contains('.') {
//return mod.replace('.', os.PathSeparator)
return mod.replace('.', '/')
}
return mod
}
// 'strings' => 'VROOT/vlib/strings'
// 'installed_mod' => '~/.vmodules/installed_mod'
// 'local_mod' => '/path/to/current/dir/local_mod'

8
compiler/parser.v
View File

@ -96,7 +96,7 @@ const (
// new parser from string. unique id specified in `id`.
// tip: use a hashing function to auto generate `id` from `text` eg. sha1.hexhash(text)
fn (v mut V) new_parser_string(text string, id string) Parser {
fn (v mut V) new_parser_from_string(text string, id string) Parser {
mut p := v.new_parser(new_scanner(text), id)
p.scan_tokens()
v.add_parser(p)
@ -104,7 +104,7 @@ fn (v mut V) new_parser_string(text string, id string) Parser {
}
// new parser from file.
fn (v mut V) new_parser_file(path string) Parser {
fn (v mut V) new_parser_from_file(path string) Parser {
//println('new_parser("$path")')
mut path_pcguard := ''
mut path_platform := '.v'
@ -505,7 +505,9 @@ fn (p mut Parser) const_decl() {
if p.first_pass() && p.table.known_const(name) {
p.error('redefinition of `$name`')
}
if p.first_pass() {
p.table.register_const(name, typ, p.mod)
}
if p.pass == .main {
// TODO hack
// cur_line has const's value right now. if it's just a number, then optimize generation:
@ -898,6 +900,7 @@ if p.scanner.line_comment != '' {
}
[inline]
fn (p &Parser) first_pass() bool {
return p.pass == .decl
}
@ -1221,6 +1224,7 @@ fn (p mut Parser) gen(s string) {
// Generate V header from V source
fn (p mut Parser) vh_genln(s string) {
//println('vh $s')
p.vh_lines << s
}

1
vlib/builtin/builtin.v
View File

@ -9,7 +9,6 @@ fn C.memmove(byteptr, byteptr, int)
//fn C.malloc(int) byteptr
fn C.realloc(byteptr, int) byteptr
pub fn exit(code int) {
C.exit(code)
}

23
vlib/builtin/string.v
View File

@ -69,7 +69,7 @@ fn todo() { }
// String data is reused, not copied.
pub fn tos(s byteptr, len int) string {
// This should never happen.
if isnil(s) {
if s == 0 {
panic('tos(): nil string')
}
return string {
@ -79,7 +79,7 @@ pub fn tos(s byteptr, len int) string {
}
pub fn tos_clone(s byteptr) string {
if isnil(s) {
if s == 0 {
panic('tos: nil string')
}
return tos2(s).clone()
@ -88,12 +88,23 @@ pub fn tos_clone(s byteptr) string {
// Same as `tos`, but calculates the length. Called by `string(bytes)` casts.
// Used only internally.
fn tos2(s byteptr) string {
if isnil(s) {
if s == 0 {
panic('tos2: nil string')
}
len := vstrlen(s)
res := tos(s, len)
return res
return string {
str: s
len: vstrlen(s)
}
}
fn tos3(s *C.char) string {
if s == 0 {
panic('tos3: nil string')
}
return string {
str: byteptr(s)
len: C.strlen(s)
}
}
pub fn (a string) clone() string {

2
vlib/strings/builder.v
View File

@ -13,7 +13,7 @@ pub:
pub fn new_builder(initial_size int) Builder {
return Builder {
//buf: _make(0, initial_size, sizeof(byte))
// buf: _make(0, initial_size, sizeof(byte))
}
}