vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

parser: cached tokens (second step)

pull/2126/head
Alexander Medvednikov 2019-09-26 05:28:43 +03:00
parent a45255337d
commit da43267e09
8 changed files with 149 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
compiler/comptime.v
View File

@ -7,6 +7,7 @@ module main
import (
vweb.tmpl // for `$vweb_html()`
os
strings
)
fn (p mut Parser) comp_time() {
@ -244,21 +245,47 @@ fn (p mut Parser) gen_array_str(typ Type) {
!p.table.type_has_method(elm_type2, 'str') {
p.error('cant print ${elm_type}[], unhandled print of ${elm_type}')
}
p.cgen.fns << '
string ${typ.name}_str($typ.name a) {
strings__Builder sb = strings__new_builder(a.len * 3);
strings__Builder_write(&sb, tos2("[")) ;
for (int i = 0; i < a.len; i++) {
strings__Builder_write(&sb, ${elm_type}_str( (($elm_type *) a.data)[i]));
if (i < a.len - 1) {
strings__Builder_write(&sb, tos2(", ")) ;
p.v.vgen_file.writeln('
fn (a $typ.name) str() string {
mut sb := strings.new_builder(a.len * 3)
sb.write("[")
for i, elm in a {
sb.write(elm.str())
if i < a.len - 1 {
sb.write(", ")
}
}
sb.write("]")
return sb.str()
}
strings__Builder_write(&sb, tos2("]")) ;
return strings__Builder_str(sb);
} '
')
p.cgen.fns << 'string ${typ.name}_str();'
}
// `Foo { bar: 3, baz: 'hi' }` => '{ bar: 3, baz: "hi" }'
fn (p mut Parser) gen_struct_str(typ Type) {
p.add_method(typ.name, Fn{
name: 'str'
typ: 'string'
args: [Var{typ: typ.name, is_arg:true}]
is_method: true
is_public: true
receiver_typ: typ.name
})
mut sb := strings.new_builder(typ.fields.len * 20)
sb.writeln('fn (a $typ.name) str() string {\nreturn')
sb.writeln("'{")
for field in typ.fields {
sb.writeln('\t$field.name: \$a.${field.name}')
}
sb.writeln("\n}'")
sb.writeln('}')
p.v.vgen_file.writeln(sb.str())
// Need to manually add the definition to `fns` so that it stays
// at the top of the file.
// This function will get parsee by V after the main pass.
p.cgen.fns << 'string ${typ.name}_str();'
}

32
compiler/fn.v
View File

@ -127,8 +127,12 @@ fn (p mut Parser) register_var(v Var) {
fn (p mut Parser) clear_vars() {
// shared a := [1, 2, 3]
p.var_idx = 0
p.local_vars.free()
p.local_vars = []Var
if p.local_vars.len > 0 {
if p.pref.autofree {
p.local_vars.free()
}
p.local_vars = []Var
}
}
// vlib header file?
@ -173,7 +177,8 @@ fn (p mut Parser) fn_decl() {
p.error('invalid receiver type `$receiver_typ` (`$receiver_typ` is an interface)')
}
// Don't allow modifying types from a different module
if !p.first_pass() && !p.builtin_mod && T.mod != p.mod {
if !p.first_pass() && !p.builtin_mod && T.mod != p.mod &&
!p.fileis(vgen_file_name) { // allow .str() on builtin arrays
println('T.mod=$T.mod')
println('p.mod=$p.mod')
p.error('cannot define new methods on non-local type `$receiver_typ`')
@ -918,21 +923,13 @@ fn (p mut Parser) fn_call_args(f mut Fn) &Fn {
p.cgen.set_placeholder(ph, '${typ}_str(')
p.gen(')')
continue
}
} else if T.cat == .struct_ {
p.gen_struct_str(T)
p.cgen.set_placeholder(ph, '${typ}_str(')
p.gen(')')
continue
}
error_msg := ('`$typ` needs to have method `str() string` to be printable')
if T.fields.len > 0 {
mut index := p.cgen.cur_line.len - 1
for index > 0 && p.cgen.cur_line[index - 1] != `(` { index-- }
name := p.cgen.cur_line.right(index + 1)
if name == '}' {
p.error(error_msg)
}
p.cgen.resetln(p.cgen.cur_line.left(index))
p.scanner.create_type_string(T, name)
p.cgen.cur_line.replace(typ, '')
p.next()
return p.fn_call_args(mut f)
}
p.error(error_msg)
}
p.cgen.set_placeholder(ph, '${typ}_str(')
@ -1029,7 +1026,6 @@ fn (p mut Parser) fn_call_args(f mut Fn) &Fn {
p.error('wrong number of arguments for fn `$f.name`: expected $f.args.len, but got more')
}
p.check(.rpar)
// p.gen(')')
return f // TODO is return f right?
}

54
compiler/main.v
View File

@ -74,6 +74,7 @@ mut:
vroot string
mod string // module being built with -lib
parsers []Parser
vgen_file os.File
}
struct Preferences {
@ -153,6 +154,8 @@ fn main() {
vfmt(args)
return
}
// Construct the V object from command line arguments
mut v := new_v(args)
if args.join(' ').contains(' test v') {
@ -223,13 +226,9 @@ fn (v mut V) compile() {
if os.user_os() != 'windows' && v.os == .msvc {
verror('Cannot build with msvc on ${os.user_os()}')
}
mut cgen := v.cgen
cgen.genln('// Generated by V')
// Add builtin parsers
for i, file in v.files {
// v.parsers << v.new_parser(file)
}
if v.pref.is_verbose {
println('all .v files before:')
println(v.files)
@ -239,11 +238,24 @@ fn (v mut V) compile() {
println('all .v files:')
println(v.files)
}
if v.pref.is_debug {
println('\nparsers:')
for q in v.parsers {
println(q.file_name)
}
println('\nfiles:')
for q in v.files {
println(q)
}
}
// First pass (declarations)
for file in v.files {
mut p := v.new_parser(file)
p.parse(.decl)
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
for i, p in v.parsers {
if p.file_path == file {
v.parsers[i].parse(.decl)
break
}
}
}
// Main pass
cgen.pass = Pass.main
@ -279,13 +291,15 @@ fn (v mut V) compile() {
v.dir.contains('/ui'))) {
cgen.genln('id defaultFont = 0; // main.v')
}
// We need the cjson header for all the json decoding user will do in default mode
// We need the cjson header for all the json decoding that will be done in
// default mode
if v.pref.build_mode == .default_mode {
if imports_json {
cgen.genln('#include "cJSON.h"')
}
}
if v.pref.build_mode == .embed_vlib || v.pref.build_mode == .default_mode {
//if v.pref.build_mode in [.embed_vlib, .default_mode] {
// If we declare these for all modes, then when running `v a.v` we'll get
// `/usr/bin/ld: multiple definition of 'total_m'`
// TODO
@ -294,7 +308,7 @@ fn (v mut V) compile() {
$if !js {
cgen.genln('int g_test_ok = 1; ')
}
if 'json' in v.table.imports {
if imports_json {
cgen.genln('
#define js_get(object, key) cJSON_GetObjectItemCaseSensitive((object), (key))
')
@ -307,8 +321,12 @@ fn (v mut V) compile() {
cgen.genln('this line will be replaced with definitions')
defs_pos := cgen.lines.len - 1
for file in v.files {
mut p := v.new_parser(file)
p.parse(.main)
for i, p in v.parsers {
if p.file_path == file {
v.parsers[i].parse(.main)
break
}
}
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
// p.g.gen_x64()
// Format all files (don't format automatically generated vlib headers)
@ -316,6 +334,10 @@ fn (v mut V) compile() {
// new vfmt is not ready yet
}
}
// Close the file with generated V code (str() methods etc) and parse it
v.vgen_file.close()
mut vgen_parser := v.new_parser(vgen_file_name)
vgen_parser.parse(.main)
v.log('Done parsing.')
// Write everything
mut d := strings.new_builder(10000)// Avoid unnecessary allocations
@ -335,8 +357,7 @@ fn (v mut V) compile() {
d.writeln('; // Prof counters:')
d.writeln(v.prof_counters())
}
dd := d.str()
cgen.lines[defs_pos] = dd// TODO `def.str()` doesn't compile
cgen.lines[defs_pos] = d.str()
v.generate_main()
v.generate_hot_reload_code()
if v.pref.is_verbose {
@ -716,6 +737,10 @@ fn (v &V) log(s string) {
}
fn new_v(args[]string) &V {
os.rm(vgen_file_name)
vgen_file := os.open_append(vgen_file_name) or { panic(err) }
vgen_file.writeln('module main\nimport strings')
joined_args := args.join(' ')
target_os := get_arg(joined_args, 'os', '')
mut out_name := get_arg(joined_args, 'o', 'a.out')
@ -907,6 +932,7 @@ fn new_v(args[]string) &V {
vroot: vroot
pref: pref
mod: mod
vgen_file: vgen_file
}
}

35
compiler/parser.v
View File

@ -9,12 +9,17 @@ import (
strings
)
const (
vgen_file_name = 'vgen.tmp'
)
// TODO rename to Token
// TODO rename enum Token to TokenType
struct Tok {
tok Token
lit string
line_nr int
tok Token
lit string
line_nr int
name_idx int // name table index for O(1) lookup
// col int
}
@ -103,6 +108,8 @@ fn (v mut V) new_parser(path string) Parser {
break
}
}
//vgen_file := os.open_append(vgen_file_name) or { panic(err) }
mut p := Parser {
v: v
@ -120,7 +127,6 @@ fn (v mut V) new_parser(path string) Parser {
os: v.os
vroot: v.vroot
local_vars: [Var{}].repeat(MaxLocalVars)
}
$if js {
p.is_js = true
@ -144,7 +150,15 @@ fn (v mut V) new_parser(path string) Parser {
break
}
}
v.add_parser(p)
/*
if !(p in v.parsers) {
v.parsers << p
}
*/
//p.next()
//p.scanner.debug_tokens()
return p
@ -202,6 +216,7 @@ fn (p & Parser) peek() Token {
/*
fn (p mut Parser) next_old() {
p.prev_tok2 = p.prev_tok
p.prev_tok = p.tok
@ -210,6 +225,7 @@ fn (p mut Parser) next_old() {
p.tok = res.tok
p.lit = res.lit
}
*/
fn (p &Parser) log(s string) {
/*
@ -423,7 +439,7 @@ fn (p mut Parser) import_statement() {
if p.tok != .name {
p.error('bad import format')
}
if p.peek() == .number && p.scanner.text[p.scanner.pos + 1] == `.` {
if p.peek() == .number { // && p.scanner.text[p.scanner.pos + 1] == `.` {
p.error('bad import format. module/submodule names cannot begin with a number')
}
mut mod := p.check_name().trim_space()
@ -842,15 +858,18 @@ fn (p mut Parser) check(expected Token) {
print_backtrace()
p.error(s)
}
/*
if expected == .rcbr {
p.fmt_dec()
}
p.fgen(p.strtok())
// vfmt: increase indentation on `{` unless it's `{}`
// TODO
if expected == .lcbr && p.scanner.pos + 1 < p.scanner.text.len && p.scanner.text[p.scanner.pos + 1] != `}` {
p.fgenln('')
p.fmt_inc()
}
*/
p.next()
if p.scanner.line_comment != '' {
@ -3896,6 +3915,10 @@ fn (p mut Parser) check_and_register_used_imported_type(typ_name string) {
}
fn (p mut Parser) check_unused_imports() {
// Don't run in the generated V file with `.str()`
if p.fileis(vgen_file_name) {
return
}
mut output := ''
for alias, mod in p.import_table.imports {
if !p.import_table.is_used_import(alias) {

56
compiler/scanner.v
View File

@ -38,11 +38,11 @@ mut:
fn new_scanner(file_path string) &Scanner {
if !os.file_exists(file_path) {
verror('"$file_path" doesn\'t exist')
verror("$file_path doesn't exist")
}
mut raw_text := os.read_file(file_path) or {
verror('scanner: failed to open "$file_path"')
verror('scanner: failed to open $file_path')
return 0
}
@ -660,14 +660,15 @@ fn (s &Scanner) error(msg string) {
println(pointerline)
}
fullpath := os.realpath( s.file_path )
_ = fullpath
// The filepath:line:col: format is the default C compiler
// error output format. It allows editors and IDE's like
// emacs to quickly find the errors in the output
// and jump to their source with a keyboard shortcut.
// Using only the filename leads to inability of IDE/editors
// to find the source file, when it is in another folder.
//println('${s.file_path}:${s.line_nr + 1}:${column+1}: $msg')
println('${fullpath}:${s.line_nr + 1}:${column+1}: $msg')
println('${s.file_path}:${s.line_nr + 1}:${column+1}: $msg')
//println('${fullpath}:${s.line_nr + 1}:${column+1}: $msg')
exit(1)
}
@ -834,53 +835,8 @@ fn is_nl(c byte) bool {
return c == `\r` || c == `\n`
}
fn (s &Scanner) get_opening_bracket() int {
mut pos := s.pos
mut parentheses := 0
mut inside_string := false
for pos > 0 && s.text[pos] != `\n` {
if s.text[pos] == `)` && !inside_string {
parentheses++
}
if s.text[pos] == `(` && !inside_string {
parentheses--
}
if s.text[pos] == `\'` && s.text[pos - 1] != `\\` && s.text[pos - 1] != `\`` { // ` // apostrophe balance comment. do not remove
inside_string = !inside_string
}
if parentheses == 0 {
break
}
pos--
}
return pos
}
// Foo { bar: 3, baz: 'hi' } => '{ bar: 3, baz: "hi" }'
fn (s mut Scanner) create_type_string(T Type, name string) {
line := s.line_nr
inside_string := s.inside_string
mut newtext := '\'{ '
start := s.get_opening_bracket() + 1
end := s.pos
for i, field in T.fields {
if i != 0 {
newtext += ', '
}
newtext += '$field.name: ' + '$${name}.${field.name}'
}
newtext += ' }\''
s.text = s.text.substr(0, start) + newtext + s.text.substr(end, s.text.len)
s.pos = start - 2
s.line_nr = line
s.inside_string = inside_string
}
fn contains_capital(s string) bool {
// for c in s {
for i := 0; i < s.len; i++ {
c := s[i]
for c in s {
if c >= `A` && c <= `Z` {
return true
}

15
compiler/table.v
View File

@ -20,8 +20,23 @@ mut:
cflags []CFlag // ['-framework Cocoa', '-lglfw3']
fn_cnt int //atomic
obfuscate bool
//names []Name
}
/*
enum NameCategory {
constant
mod
var
typ
}
struct Name {
cat NameCategory
}
*/
struct GenTable {
fn_name string
mut:

6
compiler/tests/str_gen_test.v
View File

@ -1,9 +1,11 @@
struct Foo {
a int
number int
str string
f f64
}
fn test_array_str() {
f := Foo{34}
f := Foo{34, 'hello', 1.2}
println(f)
//s := f.str()
//println(s)

2
september.plan
View File

@ -7,7 +7,7 @@
- remove all compiler memory leaks
- fix child <T> function calls
+ fix non-ascii rendering in gg (ä, å, etc)
- cache all tokens once
+ cache all tokens once
- enable vfmt
- bring back vdoc and regenerate all module docs
- optimize the parser (reduce map lookups)