parser: cached tokens (second step)
parent
a45255337d
commit
da43267e09
|
@ -7,6 +7,7 @@ module main
|
|||
import (
|
||||
vweb.tmpl // for `$vweb_html()`
|
||||
os
|
||||
strings
|
||||
)
|
||||
|
||||
fn (p mut Parser) comp_time() {
|
||||
|
@ -244,21 +245,47 @@ fn (p mut Parser) gen_array_str(typ Type) {
|
|||
!p.table.type_has_method(elm_type2, 'str') {
|
||||
p.error('cant print ${elm_type}[], unhandled print of ${elm_type}')
|
||||
}
|
||||
p.cgen.fns << '
|
||||
string ${typ.name}_str($typ.name a) {
|
||||
strings__Builder sb = strings__new_builder(a.len * 3);
|
||||
strings__Builder_write(&sb, tos2("[")) ;
|
||||
for (int i = 0; i < a.len; i++) {
|
||||
strings__Builder_write(&sb, ${elm_type}_str( (($elm_type *) a.data)[i]));
|
||||
|
||||
if (i < a.len - 1) {
|
||||
strings__Builder_write(&sb, tos2(", ")) ;
|
||||
|
||||
p.v.vgen_file.writeln('
|
||||
fn (a $typ.name) str() string {
|
||||
mut sb := strings.new_builder(a.len * 3)
|
||||
sb.write("[")
|
||||
for i, elm in a {
|
||||
sb.write(elm.str())
|
||||
if i < a.len - 1 {
|
||||
sb.write(", ")
|
||||
}
|
||||
}
|
||||
sb.write("]")
|
||||
return sb.str()
|
||||
}
|
||||
strings__Builder_write(&sb, tos2("]")) ;
|
||||
return strings__Builder_str(sb);
|
||||
} '
|
||||
')
|
||||
p.cgen.fns << 'string ${typ.name}_str();'
|
||||
}
|
||||
|
||||
// `Foo { bar: 3, baz: 'hi' }` => '{ bar: 3, baz: "hi" }'
|
||||
fn (p mut Parser) gen_struct_str(typ Type) {
|
||||
p.add_method(typ.name, Fn{
|
||||
name: 'str'
|
||||
typ: 'string'
|
||||
args: [Var{typ: typ.name, is_arg:true}]
|
||||
is_method: true
|
||||
is_public: true
|
||||
receiver_typ: typ.name
|
||||
})
|
||||
|
||||
mut sb := strings.new_builder(typ.fields.len * 20)
|
||||
sb.writeln('fn (a $typ.name) str() string {\nreturn')
|
||||
sb.writeln("'{")
|
||||
for field in typ.fields {
|
||||
sb.writeln('\t$field.name: \$a.${field.name}')
|
||||
}
|
||||
sb.writeln("\n}'")
|
||||
sb.writeln('}')
|
||||
p.v.vgen_file.writeln(sb.str())
|
||||
// Need to manually add the definition to `fns` so that it stays
|
||||
// at the top of the file.
|
||||
// This function will get parsee by V after the main pass.
|
||||
p.cgen.fns << 'string ${typ.name}_str();'
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -127,8 +127,12 @@ fn (p mut Parser) register_var(v Var) {
|
|||
fn (p mut Parser) clear_vars() {
|
||||
// shared a := [1, 2, 3]
|
||||
p.var_idx = 0
|
||||
p.local_vars.free()
|
||||
p.local_vars = []Var
|
||||
if p.local_vars.len > 0 {
|
||||
if p.pref.autofree {
|
||||
p.local_vars.free()
|
||||
}
|
||||
p.local_vars = []Var
|
||||
}
|
||||
}
|
||||
|
||||
// vlib header file?
|
||||
|
@ -173,7 +177,8 @@ fn (p mut Parser) fn_decl() {
|
|||
p.error('invalid receiver type `$receiver_typ` (`$receiver_typ` is an interface)')
|
||||
}
|
||||
// Don't allow modifying types from a different module
|
||||
if !p.first_pass() && !p.builtin_mod && T.mod != p.mod {
|
||||
if !p.first_pass() && !p.builtin_mod && T.mod != p.mod &&
|
||||
!p.fileis(vgen_file_name) { // allow .str() on builtin arrays
|
||||
println('T.mod=$T.mod')
|
||||
println('p.mod=$p.mod')
|
||||
p.error('cannot define new methods on non-local type `$receiver_typ`')
|
||||
|
@ -918,21 +923,13 @@ fn (p mut Parser) fn_call_args(f mut Fn) &Fn {
|
|||
p.cgen.set_placeholder(ph, '${typ}_str(')
|
||||
p.gen(')')
|
||||
continue
|
||||
}
|
||||
} else if T.cat == .struct_ {
|
||||
p.gen_struct_str(T)
|
||||
p.cgen.set_placeholder(ph, '${typ}_str(')
|
||||
p.gen(')')
|
||||
continue
|
||||
}
|
||||
error_msg := ('`$typ` needs to have method `str() string` to be printable')
|
||||
if T.fields.len > 0 {
|
||||
mut index := p.cgen.cur_line.len - 1
|
||||
for index > 0 && p.cgen.cur_line[index - 1] != `(` { index-- }
|
||||
name := p.cgen.cur_line.right(index + 1)
|
||||
if name == '}' {
|
||||
p.error(error_msg)
|
||||
}
|
||||
p.cgen.resetln(p.cgen.cur_line.left(index))
|
||||
p.scanner.create_type_string(T, name)
|
||||
p.cgen.cur_line.replace(typ, '')
|
||||
p.next()
|
||||
return p.fn_call_args(mut f)
|
||||
}
|
||||
p.error(error_msg)
|
||||
}
|
||||
p.cgen.set_placeholder(ph, '${typ}_str(')
|
||||
|
@ -1029,7 +1026,6 @@ fn (p mut Parser) fn_call_args(f mut Fn) &Fn {
|
|||
p.error('wrong number of arguments for fn `$f.name`: expected $f.args.len, but got more')
|
||||
}
|
||||
p.check(.rpar)
|
||||
// p.gen(')')
|
||||
return f // TODO is return f right?
|
||||
}
|
||||
|
||||
|
|
|
@ -74,6 +74,7 @@ mut:
|
|||
vroot string
|
||||
mod string // module being built with -lib
|
||||
parsers []Parser
|
||||
vgen_file os.File
|
||||
}
|
||||
|
||||
struct Preferences {
|
||||
|
@ -153,6 +154,8 @@ fn main() {
|
|||
vfmt(args)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
// Construct the V object from command line arguments
|
||||
mut v := new_v(args)
|
||||
if args.join(' ').contains(' test v') {
|
||||
|
@ -223,13 +226,9 @@ fn (v mut V) compile() {
|
|||
if os.user_os() != 'windows' && v.os == .msvc {
|
||||
verror('Cannot build with msvc on ${os.user_os()}')
|
||||
}
|
||||
|
||||
|
||||
mut cgen := v.cgen
|
||||
cgen.genln('// Generated by V')
|
||||
// Add builtin parsers
|
||||
for i, file in v.files {
|
||||
// v.parsers << v.new_parser(file)
|
||||
}
|
||||
if v.pref.is_verbose {
|
||||
println('all .v files before:')
|
||||
println(v.files)
|
||||
|
@ -239,11 +238,24 @@ fn (v mut V) compile() {
|
|||
println('all .v files:')
|
||||
println(v.files)
|
||||
}
|
||||
if v.pref.is_debug {
|
||||
println('\nparsers:')
|
||||
for q in v.parsers {
|
||||
println(q.file_name)
|
||||
}
|
||||
println('\nfiles:')
|
||||
for q in v.files {
|
||||
println(q)
|
||||
}
|
||||
}
|
||||
// First pass (declarations)
|
||||
for file in v.files {
|
||||
mut p := v.new_parser(file)
|
||||
p.parse(.decl)
|
||||
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
|
||||
for i, p in v.parsers {
|
||||
if p.file_path == file {
|
||||
v.parsers[i].parse(.decl)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Main pass
|
||||
cgen.pass = Pass.main
|
||||
|
@ -279,13 +291,15 @@ fn (v mut V) compile() {
|
|||
v.dir.contains('/ui'))) {
|
||||
cgen.genln('id defaultFont = 0; // main.v')
|
||||
}
|
||||
// We need the cjson header for all the json decoding user will do in default mode
|
||||
// We need the cjson header for all the json decoding that will be done in
|
||||
// default mode
|
||||
if v.pref.build_mode == .default_mode {
|
||||
if imports_json {
|
||||
cgen.genln('#include "cJSON.h"')
|
||||
}
|
||||
}
|
||||
if v.pref.build_mode == .embed_vlib || v.pref.build_mode == .default_mode {
|
||||
//if v.pref.build_mode in [.embed_vlib, .default_mode] {
|
||||
// If we declare these for all modes, then when running `v a.v` we'll get
|
||||
// `/usr/bin/ld: multiple definition of 'total_m'`
|
||||
// TODO
|
||||
|
@ -294,7 +308,7 @@ fn (v mut V) compile() {
|
|||
$if !js {
|
||||
cgen.genln('int g_test_ok = 1; ')
|
||||
}
|
||||
if 'json' in v.table.imports {
|
||||
if imports_json {
|
||||
cgen.genln('
|
||||
#define js_get(object, key) cJSON_GetObjectItemCaseSensitive((object), (key))
|
||||
')
|
||||
|
@ -307,8 +321,12 @@ fn (v mut V) compile() {
|
|||
cgen.genln('this line will be replaced with definitions')
|
||||
defs_pos := cgen.lines.len - 1
|
||||
for file in v.files {
|
||||
mut p := v.new_parser(file)
|
||||
p.parse(.main)
|
||||
for i, p in v.parsers {
|
||||
if p.file_path == file {
|
||||
v.parsers[i].parse(.main)
|
||||
break
|
||||
}
|
||||
}
|
||||
//if p.pref.autofree { p.scanner.text.free() free(p.scanner) }
|
||||
// p.g.gen_x64()
|
||||
// Format all files (don't format automatically generated vlib headers)
|
||||
|
@ -316,6 +334,10 @@ fn (v mut V) compile() {
|
|||
// new vfmt is not ready yet
|
||||
}
|
||||
}
|
||||
// Close the file with generated V code (str() methods etc) and parse it
|
||||
v.vgen_file.close()
|
||||
mut vgen_parser := v.new_parser(vgen_file_name)
|
||||
vgen_parser.parse(.main)
|
||||
v.log('Done parsing.')
|
||||
// Write everything
|
||||
mut d := strings.new_builder(10000)// Avoid unnecessary allocations
|
||||
|
@ -335,8 +357,7 @@ fn (v mut V) compile() {
|
|||
d.writeln('; // Prof counters:')
|
||||
d.writeln(v.prof_counters())
|
||||
}
|
||||
dd := d.str()
|
||||
cgen.lines[defs_pos] = dd// TODO `def.str()` doesn't compile
|
||||
cgen.lines[defs_pos] = d.str()
|
||||
v.generate_main()
|
||||
v.generate_hot_reload_code()
|
||||
if v.pref.is_verbose {
|
||||
|
@ -716,6 +737,10 @@ fn (v &V) log(s string) {
|
|||
}
|
||||
|
||||
fn new_v(args[]string) &V {
|
||||
os.rm(vgen_file_name)
|
||||
vgen_file := os.open_append(vgen_file_name) or { panic(err) }
|
||||
vgen_file.writeln('module main\nimport strings')
|
||||
|
||||
joined_args := args.join(' ')
|
||||
target_os := get_arg(joined_args, 'os', '')
|
||||
mut out_name := get_arg(joined_args, 'o', 'a.out')
|
||||
|
@ -907,6 +932,7 @@ fn new_v(args[]string) &V {
|
|||
vroot: vroot
|
||||
pref: pref
|
||||
mod: mod
|
||||
vgen_file: vgen_file
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -9,12 +9,17 @@ import (
|
|||
strings
|
||||
)
|
||||
|
||||
const (
|
||||
vgen_file_name = 'vgen.tmp'
|
||||
)
|
||||
|
||||
// TODO rename to Token
|
||||
// TODO rename enum Token to TokenType
|
||||
struct Tok {
|
||||
tok Token
|
||||
lit string
|
||||
line_nr int
|
||||
tok Token
|
||||
lit string
|
||||
line_nr int
|
||||
name_idx int // name table index for O(1) lookup
|
||||
// col int
|
||||
}
|
||||
|
||||
|
@ -103,6 +108,8 @@ fn (v mut V) new_parser(path string) Parser {
|
|||
break
|
||||
}
|
||||
}
|
||||
|
||||
//vgen_file := os.open_append(vgen_file_name) or { panic(err) }
|
||||
|
||||
mut p := Parser {
|
||||
v: v
|
||||
|
@ -120,7 +127,6 @@ fn (v mut V) new_parser(path string) Parser {
|
|||
os: v.os
|
||||
vroot: v.vroot
|
||||
local_vars: [Var{}].repeat(MaxLocalVars)
|
||||
|
||||
}
|
||||
$if js {
|
||||
p.is_js = true
|
||||
|
@ -144,7 +150,15 @@ fn (v mut V) new_parser(path string) Parser {
|
|||
break
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
v.add_parser(p)
|
||||
/*
|
||||
if !(p in v.parsers) {
|
||||
v.parsers << p
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
//p.next()
|
||||
//p.scanner.debug_tokens()
|
||||
return p
|
||||
|
@ -202,6 +216,7 @@ fn (p & Parser) peek() Token {
|
|||
|
||||
|
||||
|
||||
/*
|
||||
fn (p mut Parser) next_old() {
|
||||
p.prev_tok2 = p.prev_tok
|
||||
p.prev_tok = p.tok
|
||||
|
@ -210,6 +225,7 @@ fn (p mut Parser) next_old() {
|
|||
p.tok = res.tok
|
||||
p.lit = res.lit
|
||||
}
|
||||
*/
|
||||
|
||||
fn (p &Parser) log(s string) {
|
||||
/*
|
||||
|
@ -423,7 +439,7 @@ fn (p mut Parser) import_statement() {
|
|||
if p.tok != .name {
|
||||
p.error('bad import format')
|
||||
}
|
||||
if p.peek() == .number && p.scanner.text[p.scanner.pos + 1] == `.` {
|
||||
if p.peek() == .number { // && p.scanner.text[p.scanner.pos + 1] == `.` {
|
||||
p.error('bad import format. module/submodule names cannot begin with a number')
|
||||
}
|
||||
mut mod := p.check_name().trim_space()
|
||||
|
@ -842,15 +858,18 @@ fn (p mut Parser) check(expected Token) {
|
|||
print_backtrace()
|
||||
p.error(s)
|
||||
}
|
||||
/*
|
||||
if expected == .rcbr {
|
||||
p.fmt_dec()
|
||||
}
|
||||
p.fgen(p.strtok())
|
||||
// vfmt: increase indentation on `{` unless it's `{}`
|
||||
// TODO
|
||||
if expected == .lcbr && p.scanner.pos + 1 < p.scanner.text.len && p.scanner.text[p.scanner.pos + 1] != `}` {
|
||||
p.fgenln('')
|
||||
p.fmt_inc()
|
||||
}
|
||||
*/
|
||||
p.next()
|
||||
|
||||
if p.scanner.line_comment != '' {
|
||||
|
@ -3896,6 +3915,10 @@ fn (p mut Parser) check_and_register_used_imported_type(typ_name string) {
|
|||
}
|
||||
|
||||
fn (p mut Parser) check_unused_imports() {
|
||||
// Don't run in the generated V file with `.str()`
|
||||
if p.fileis(vgen_file_name) {
|
||||
return
|
||||
}
|
||||
mut output := ''
|
||||
for alias, mod in p.import_table.imports {
|
||||
if !p.import_table.is_used_import(alias) {
|
||||
|
|
|
@ -38,11 +38,11 @@ mut:
|
|||
|
||||
fn new_scanner(file_path string) &Scanner {
|
||||
if !os.file_exists(file_path) {
|
||||
verror('"$file_path" doesn\'t exist')
|
||||
verror("$file_path doesn't exist")
|
||||
}
|
||||
|
||||
mut raw_text := os.read_file(file_path) or {
|
||||
verror('scanner: failed to open "$file_path"')
|
||||
verror('scanner: failed to open $file_path')
|
||||
return 0
|
||||
}
|
||||
|
||||
|
@ -660,14 +660,15 @@ fn (s &Scanner) error(msg string) {
|
|||
println(pointerline)
|
||||
}
|
||||
fullpath := os.realpath( s.file_path )
|
||||
_ = fullpath
|
||||
// The filepath:line:col: format is the default C compiler
|
||||
// error output format. It allows editors and IDE's like
|
||||
// emacs to quickly find the errors in the output
|
||||
// and jump to their source with a keyboard shortcut.
|
||||
// Using only the filename leads to inability of IDE/editors
|
||||
// to find the source file, when it is in another folder.
|
||||
//println('${s.file_path}:${s.line_nr + 1}:${column+1}: $msg')
|
||||
println('${fullpath}:${s.line_nr + 1}:${column+1}: $msg')
|
||||
println('${s.file_path}:${s.line_nr + 1}:${column+1}: $msg')
|
||||
//println('${fullpath}:${s.line_nr + 1}:${column+1}: $msg')
|
||||
exit(1)
|
||||
}
|
||||
|
||||
|
@ -834,53 +835,8 @@ fn is_nl(c byte) bool {
|
|||
return c == `\r` || c == `\n`
|
||||
}
|
||||
|
||||
fn (s &Scanner) get_opening_bracket() int {
|
||||
mut pos := s.pos
|
||||
mut parentheses := 0
|
||||
mut inside_string := false
|
||||
|
||||
for pos > 0 && s.text[pos] != `\n` {
|
||||
if s.text[pos] == `)` && !inside_string {
|
||||
parentheses++
|
||||
}
|
||||
if s.text[pos] == `(` && !inside_string {
|
||||
parentheses--
|
||||
}
|
||||
if s.text[pos] == `\'` && s.text[pos - 1] != `\\` && s.text[pos - 1] != `\`` { // ` // apostrophe balance comment. do not remove
|
||||
inside_string = !inside_string
|
||||
}
|
||||
if parentheses == 0 {
|
||||
break
|
||||
}
|
||||
pos--
|
||||
}
|
||||
return pos
|
||||
}
|
||||
|
||||
// Foo { bar: 3, baz: 'hi' } => '{ bar: 3, baz: "hi" }'
|
||||
fn (s mut Scanner) create_type_string(T Type, name string) {
|
||||
line := s.line_nr
|
||||
inside_string := s.inside_string
|
||||
mut newtext := '\'{ '
|
||||
start := s.get_opening_bracket() + 1
|
||||
end := s.pos
|
||||
for i, field in T.fields {
|
||||
if i != 0 {
|
||||
newtext += ', '
|
||||
}
|
||||
newtext += '$field.name: ' + '$${name}.${field.name}'
|
||||
}
|
||||
newtext += ' }\''
|
||||
s.text = s.text.substr(0, start) + newtext + s.text.substr(end, s.text.len)
|
||||
s.pos = start - 2
|
||||
s.line_nr = line
|
||||
s.inside_string = inside_string
|
||||
}
|
||||
|
||||
fn contains_capital(s string) bool {
|
||||
// for c in s {
|
||||
for i := 0; i < s.len; i++ {
|
||||
c := s[i]
|
||||
for c in s {
|
||||
if c >= `A` && c <= `Z` {
|
||||
return true
|
||||
}
|
||||
|
|
|
@ -20,8 +20,23 @@ mut:
|
|||
cflags []CFlag // ['-framework Cocoa', '-lglfw3']
|
||||
fn_cnt int //atomic
|
||||
obfuscate bool
|
||||
//names []Name
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
enum NameCategory {
|
||||
constant
|
||||
mod
|
||||
var
|
||||
typ
|
||||
}
|
||||
|
||||
struct Name {
|
||||
cat NameCategory
|
||||
}
|
||||
*/
|
||||
|
||||
struct GenTable {
|
||||
fn_name string
|
||||
mut:
|
||||
|
|
|
@ -1,9 +1,11 @@
|
|||
struct Foo {
|
||||
a int
|
||||
number int
|
||||
str string
|
||||
f f64
|
||||
}
|
||||
|
||||
fn test_array_str() {
|
||||
f := Foo{34}
|
||||
f := Foo{34, 'hello', 1.2}
|
||||
println(f)
|
||||
//s := f.str()
|
||||
//println(s)
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
- remove all compiler memory leaks
|
||||
- fix child <T> function calls
|
||||
+ fix non-ascii rendering in gg (ä, å, etc)
|
||||
- cache all tokens once
|
||||
+ cache all tokens once
|
||||
- enable vfmt
|
||||
- bring back vdoc and regenerate all module docs
|
||||
- optimize the parser (reduce map lookups)
|
||||
|
|
Loading…
Reference in New Issue