parser: cache tokens (first step)
parent
04e4018228
commit
8992707fbb
|
@ -20,7 +20,7 @@ fn (p mut Parser) comp_time() {
|
|||
}
|
||||
name := p.check_name()
|
||||
p.fspace()
|
||||
if name in SupportedPlatforms {
|
||||
if name in supported_platforms {
|
||||
ifdef_name := os_name_to_ifdef(name)
|
||||
if not {
|
||||
p.genln('#ifndef $ifdef_name')
|
||||
|
@ -42,7 +42,7 @@ fn (p mut Parser) comp_time() {
|
|||
}
|
||||
else {
|
||||
println('Supported platforms:')
|
||||
println(SupportedPlatforms)
|
||||
println(supported_platforms)
|
||||
p.error('unknown platform `$name`')
|
||||
}
|
||||
if_returns := p.returns
|
||||
|
|
|
@ -27,7 +27,7 @@ enum BuildMode {
|
|||
}
|
||||
|
||||
const (
|
||||
SupportedPlatforms = ['windows', 'mac', 'linux', 'freebsd', 'openbsd',
|
||||
supported_platforms = ['windows', 'mac', 'linux', 'freebsd', 'openbsd',
|
||||
'netbsd', 'dragonfly', 'msvc', 'android', 'js']
|
||||
ModPath = os.home_dir() + '/.vmodules/'
|
||||
)
|
||||
|
|
|
@ -9,6 +9,15 @@ import (
|
|||
strings
|
||||
)
|
||||
|
||||
// TODO rename to Token
|
||||
// TODO rename enum Token to TokenType
|
||||
struct Tok {
|
||||
tok Token
|
||||
lit string
|
||||
line_nr int
|
||||
// col int
|
||||
}
|
||||
|
||||
struct Parser {
|
||||
file_path string // "/home/user/hello.v"
|
||||
file_name string // "hello.v"
|
||||
|
@ -21,7 +30,7 @@ struct Parser {
|
|||
pref &Preferences // Preferences shared from V struct
|
||||
mut:
|
||||
scanner &Scanner
|
||||
// tokens []Token // TODO cache all tokens, right now they have to be scanned twice
|
||||
tokens []Tok
|
||||
token_idx int
|
||||
tok Token
|
||||
prev_tok Token
|
||||
|
@ -124,7 +133,19 @@ fn (v mut V) new_parser(path string) Parser {
|
|||
v.cgen.line_directives = v.pref.is_debuggable
|
||||
v.cgen.file = path
|
||||
|
||||
p.next()
|
||||
for {
|
||||
res := p.scanner.scan()
|
||||
p.tokens << Tok {
|
||||
tok: res.tok
|
||||
lit: res.lit
|
||||
line_nr: p.scanner.line_nr
|
||||
}
|
||||
if res.tok == .eof {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//p.next()
|
||||
//p.scanner.debug_tokens()
|
||||
return p
|
||||
}
|
||||
|
@ -136,6 +157,52 @@ fn (p mut Parser) set_current_fn(f Fn) {
|
|||
}
|
||||
|
||||
fn (p mut Parser) next() {
|
||||
p.prev_tok2 = p.prev_tok
|
||||
p.prev_tok = p.tok
|
||||
p.scanner.prev_tok = p.tok
|
||||
if p.token_idx >= p.tokens.len {
|
||||
p.tok = Token.eof
|
||||
p.lit = ''
|
||||
return
|
||||
}
|
||||
res := p.tokens[p.token_idx]
|
||||
p.token_idx++
|
||||
p.tok = res.tok
|
||||
p.lit = res.lit
|
||||
p.scanner.line_nr = res.line_nr
|
||||
}
|
||||
|
||||
|
||||
fn (p & Parser) peek() Token {
|
||||
if p.token_idx >= p.tokens.len - 2 {
|
||||
return Token.eof
|
||||
}
|
||||
tok := p.tokens[p.token_idx]
|
||||
return tok.tok
|
||||
/*
|
||||
// save scanner state
|
||||
pos := s.pos
|
||||
line := s.line_nr
|
||||
inside_string := s.inside_string
|
||||
inter_start := s.inter_start
|
||||
inter_end := s.inter_end
|
||||
|
||||
res := s.scan()
|
||||
tok := res.tok
|
||||
|
||||
// restore scanner state
|
||||
s.pos = pos
|
||||
s.line_nr = line
|
||||
s.inside_string = inside_string
|
||||
s.inter_start = inter_start
|
||||
s.inter_end = inter_end
|
||||
return tok
|
||||
*/
|
||||
}
|
||||
|
||||
|
||||
|
||||
fn (p mut Parser) next_old() {
|
||||
p.prev_tok2 = p.prev_tok
|
||||
p.prev_tok = p.tok
|
||||
p.scanner.prev_tok = p.tok
|
||||
|
@ -155,6 +222,8 @@ fn (p &Parser) log(s string) {
|
|||
|
||||
fn (p mut Parser) parse(pass Pass) {
|
||||
p.pass = pass
|
||||
p.token_idx = 0
|
||||
p.next()
|
||||
//p.log('\nparse() run=$p.pass file=$p.file_name tok=${p.strtok()}')// , "script_file=", script_file)
|
||||
// `module main` is not required if it's a single file program
|
||||
if p.is_script || p.pref.is_test {
|
||||
|
@ -1579,14 +1648,16 @@ fn (p mut Parser) name_expr() string {
|
|||
}
|
||||
// //////////////////////////
|
||||
// module ?
|
||||
// (Allow shadowing `gg = gg.newcontext(); gg.draw_triangle();` )
|
||||
if p.peek() == .dot && ((name == p.mod && p.table.known_mod(name)) || p.import_table.known_alias(name))
|
||||
&& !p.known_var(name) && !is_c {
|
||||
if p.peek() == .dot && ((name == p.mod && p.table.known_mod(name)) ||
|
||||
p.import_table.known_alias(name)) && !is_c &&
|
||||
!p.known_var(name) // Allow shadowing (`gg = gg.newcontext(); gg.foo()`)
|
||||
{
|
||||
mut mod := name
|
||||
// must be aliased module
|
||||
if name != p.mod && p.import_table.known_alias(name) {
|
||||
p.import_table.register_used_import(name)
|
||||
// we replaced "." with "_dot_" in p.mod for C variable names, do same here.
|
||||
// we replaced "." with "_dot_" in p.mod for C variable names,
|
||||
// do same here.
|
||||
mod = p.import_table.resolve_alias(name).replace('.', '_dot_')
|
||||
}
|
||||
p.next()
|
||||
|
@ -1596,7 +1667,8 @@ fn (p mut Parser) name_expr() string {
|
|||
name = prepend_mod(mod, name)
|
||||
}
|
||||
else if !p.table.known_type(name) && !p.known_var(name) &&
|
||||
!p.table.known_fn(name) && !p.table.known_const(name) && !is_c {
|
||||
!p.table.known_fn(name) && !p.table.known_const(name) && !is_c
|
||||
{
|
||||
name = p.prepend_mod(name)
|
||||
}
|
||||
// Variable
|
||||
|
@ -2800,10 +2872,10 @@ fn (p mut Parser) array_init() string {
|
|||
typ = val_typ
|
||||
// fixed width array initialization? (`arr := [20]byte`)
|
||||
if is_integer && p.tok == .rsbr && p.peek() == .name {
|
||||
nextc := p.scanner.text[p.scanner.pos + 1]
|
||||
//nextc := p.scanner.text[p.scanner.pos + 1]
|
||||
// TODO whitespace hack
|
||||
// Make sure there's no space in `[10]byte`
|
||||
if !nextc.is_space() {
|
||||
//if !nextc.is_space() {
|
||||
p.check(.rsbr)
|
||||
array_elem_typ := p.get_type()
|
||||
if !p.table.known_type(array_elem_typ) {
|
||||
|
@ -2816,7 +2888,7 @@ fn (p mut Parser) array_init() string {
|
|||
return '[${p.mod}__$lit]$array_elem_typ'
|
||||
}
|
||||
return '[$lit]$array_elem_typ'
|
||||
}
|
||||
//}
|
||||
}
|
||||
}
|
||||
if val_typ != typ {
|
||||
|
|
|
@ -785,26 +785,6 @@ fn (s mut Scanner) ident_char() string {
|
|||
return if c == '\'' { '\\' + c } else { c }
|
||||
}
|
||||
|
||||
fn (s mut Scanner) peek() Token {
|
||||
// save scanner state
|
||||
pos := s.pos
|
||||
line := s.line_nr
|
||||
inside_string := s.inside_string
|
||||
inter_start := s.inter_start
|
||||
inter_end := s.inter_end
|
||||
|
||||
res := s.scan()
|
||||
tok := res.tok
|
||||
|
||||
// restore scanner state
|
||||
s.pos = pos
|
||||
s.line_nr = line
|
||||
s.inside_string = inside_string
|
||||
s.inter_start = inter_start
|
||||
s.inter_end = inter_end
|
||||
return tok
|
||||
}
|
||||
|
||||
fn (s &Scanner) expect(want string, start_pos int) bool {
|
||||
end_pos := start_pos + want.len
|
||||
if start_pos < 0 || start_pos >= s.text.len {
|
||||
|
|
|
@ -37,6 +37,7 @@ fn (p mut Parser) fgenln(s string) {
|
|||
p.scanner.fgenln(s)
|
||||
}
|
||||
|
||||
/*
|
||||
fn (p mut Parser) peek() Token {
|
||||
for {
|
||||
p.cgen.line = p.scanner.line_nr + 1
|
||||
|
@ -47,6 +48,7 @@ fn (p mut Parser) peek() Token {
|
|||
}
|
||||
return .eof // TODO can never get here - v doesn't know that
|
||||
}
|
||||
*/
|
||||
|
||||
fn (p mut Parser) fmt_inc() {
|
||||
p.scanner.fmt_indent++
|
||||
|
|
Loading…
Reference in New Issue