From 0fcdd7db358bf5580ca7a073faf7c2d109654f60 Mon Sep 17 00:00:00 2001 From: Alexander Medvednikov Date: Thu, 26 Sep 2019 06:05:18 +0300 Subject: [PATCH] Revert "parser: cache tokens (first step)" This reverts commit 8992707fbbcbbe5eeee8fbd2665715726b3b4b2c. --- compiler/comptime.v | 4 +- compiler/main.v | 2 +- compiler/parser.v | 94 ++++++--------------------------------------- compiler/scanner.v | 20 ++++++++++ compiler/vfmt.v | 2 - 5 files changed, 34 insertions(+), 88 deletions(-) diff --git a/compiler/comptime.v b/compiler/comptime.v index e6b9fad910..70788f8930 100644 --- a/compiler/comptime.v +++ b/compiler/comptime.v @@ -20,7 +20,7 @@ fn (p mut Parser) comp_time() { } name := p.check_name() p.fspace() - if name in supported_platforms { + if name in SupportedPlatforms { ifdef_name := os_name_to_ifdef(name) if not { p.genln('#ifndef $ifdef_name') @@ -42,7 +42,7 @@ fn (p mut Parser) comp_time() { } else { println('Supported platforms:') - println(supported_platforms) + println(SupportedPlatforms) p.error('unknown platform `$name`') } if_returns := p.returns diff --git a/compiler/main.v b/compiler/main.v index 1741b62359..4b0171efba 100644 --- a/compiler/main.v +++ b/compiler/main.v @@ -27,7 +27,7 @@ enum BuildMode { } const ( - supported_platforms = ['windows', 'mac', 'linux', 'freebsd', 'openbsd', + SupportedPlatforms = ['windows', 'mac', 'linux', 'freebsd', 'openbsd', 'netbsd', 'dragonfly', 'msvc', 'android', 'js'] ModPath = os.home_dir() + '/.vmodules/' ) diff --git a/compiler/parser.v b/compiler/parser.v index da61c9c93b..009b454698 100644 --- a/compiler/parser.v +++ b/compiler/parser.v @@ -9,15 +9,6 @@ import ( strings ) -// TODO rename to Token -// TODO rename enum Token to TokenType -struct Tok { - tok Token - lit string - line_nr int -// col int -} - struct Parser { file_path string // "/home/user/hello.v" file_name string // "hello.v" @@ -30,7 +21,7 @@ struct Parser { pref &Preferences // Preferences shared from V struct mut: scanner &Scanner - tokens []Tok + // tokens []Token // TODO cache all tokens, right now they have to be scanned twice token_idx int tok Token prev_tok Token @@ -133,20 +124,8 @@ fn (v mut V) new_parser(path string) Parser { v.cgen.line_directives = v.pref.is_debuggable v.cgen.file = path - for { - res := p.scanner.scan() - p.tokens << Tok { - tok: res.tok - lit: res.lit - line_nr: p.scanner.line_nr - } - if res.tok == .eof { - break - } - } - - //p.next() - //p.scanner.debug_tokens() + p.next() + // p.scanner.debug_tokens() return p } @@ -157,52 +136,6 @@ fn (p mut Parser) set_current_fn(f Fn) { } fn (p mut Parser) next() { - p.prev_tok2 = p.prev_tok - p.prev_tok = p.tok - p.scanner.prev_tok = p.tok - if p.token_idx >= p.tokens.len { - p.tok = Token.eof - p.lit = '' - return - } - res := p.tokens[p.token_idx] - p.token_idx++ - p.tok = res.tok - p.lit = res.lit - p.scanner.line_nr = res.line_nr - } - - -fn (p & Parser) peek() Token { - if p.token_idx >= p.tokens.len - 2 { - return Token.eof - } - tok := p.tokens[p.token_idx] - return tok.tok -/* - // save scanner state - pos := s.pos - line := s.line_nr - inside_string := s.inside_string - inter_start := s.inter_start - inter_end := s.inter_end - - res := s.scan() - tok := res.tok - - // restore scanner state - s.pos = pos - s.line_nr = line - s.inside_string = inside_string - s.inter_start = inter_start - s.inter_end = inter_end - return tok - */ -} - - - -fn (p mut Parser) next_old() { p.prev_tok2 = p.prev_tok p.prev_tok = p.tok p.scanner.prev_tok = p.tok @@ -222,8 +155,6 @@ fn (p &Parser) log(s string) { fn (p mut Parser) parse(pass Pass) { p.pass = pass - p.token_idx = 0 - p.next() //p.log('\nparse() run=$p.pass file=$p.file_name tok=${p.strtok()}')// , "script_file=", script_file) // `module main` is not required if it's a single file program if p.is_script || p.pref.is_test { @@ -1648,16 +1579,14 @@ fn (p mut Parser) name_expr() string { } // ////////////////////////// // module ? - if p.peek() == .dot && ((name == p.mod && p.table.known_mod(name)) || - p.import_table.known_alias(name)) && !is_c && - !p.known_var(name) // Allow shadowing (`gg = gg.newcontext(); gg.foo()`) - { + // (Allow shadowing `gg = gg.newcontext(); gg.draw_triangle();` ) + if p.peek() == .dot && ((name == p.mod && p.table.known_mod(name)) || p.import_table.known_alias(name)) + && !p.known_var(name) && !is_c { mut mod := name // must be aliased module if name != p.mod && p.import_table.known_alias(name) { p.import_table.register_used_import(name) - // we replaced "." with "_dot_" in p.mod for C variable names, - // do same here. + // we replaced "." with "_dot_" in p.mod for C variable names, do same here. mod = p.import_table.resolve_alias(name).replace('.', '_dot_') } p.next() @@ -1667,8 +1596,7 @@ fn (p mut Parser) name_expr() string { name = prepend_mod(mod, name) } else if !p.table.known_type(name) && !p.known_var(name) && - !p.table.known_fn(name) && !p.table.known_const(name) && !is_c - { + !p.table.known_fn(name) && !p.table.known_const(name) && !is_c { name = p.prepend_mod(name) } // Variable @@ -2872,10 +2800,10 @@ fn (p mut Parser) array_init() string { typ = val_typ // fixed width array initialization? (`arr := [20]byte`) if is_integer && p.tok == .rsbr && p.peek() == .name { - //nextc := p.scanner.text[p.scanner.pos + 1] + nextc := p.scanner.text[p.scanner.pos + 1] // TODO whitespace hack // Make sure there's no space in `[10]byte` - //if !nextc.is_space() { + if !nextc.is_space() { p.check(.rsbr) array_elem_typ := p.get_type() if !p.table.known_type(array_elem_typ) { @@ -2888,7 +2816,7 @@ fn (p mut Parser) array_init() string { return '[${p.mod}__$lit]$array_elem_typ' } return '[$lit]$array_elem_typ' - //} + } } } if val_typ != typ { diff --git a/compiler/scanner.v b/compiler/scanner.v index a790d2409d..bbc4c0e41b 100644 --- a/compiler/scanner.v +++ b/compiler/scanner.v @@ -785,6 +785,26 @@ fn (s mut Scanner) ident_char() string { return if c == '\'' { '\\' + c } else { c } } +fn (s mut Scanner) peek() Token { + // save scanner state + pos := s.pos + line := s.line_nr + inside_string := s.inside_string + inter_start := s.inter_start + inter_end := s.inter_end + + res := s.scan() + tok := res.tok + + // restore scanner state + s.pos = pos + s.line_nr = line + s.inside_string = inside_string + s.inter_start = inter_start + s.inter_end = inter_end + return tok +} + fn (s &Scanner) expect(want string, start_pos int) bool { end_pos := start_pos + want.len if start_pos < 0 || start_pos >= s.text.len { diff --git a/compiler/vfmt.v b/compiler/vfmt.v index 69a5b05e99..74db232ac6 100644 --- a/compiler/vfmt.v +++ b/compiler/vfmt.v @@ -37,7 +37,6 @@ fn (p mut Parser) fgenln(s string) { p.scanner.fgenln(s) } -/* fn (p mut Parser) peek() Token { for { p.cgen.line = p.scanner.line_nr + 1 @@ -48,7 +47,6 @@ fn (p mut Parser) peek() Token { } return .eof // TODO can never get here - v doesn't know that } -*/ fn (p mut Parser) fmt_inc() { p.scanner.fmt_indent++