From a45255337d9f18179263885a27ea09c90733a9cc Mon Sep 17 00:00:00 2001
From: Alexander Medvednikov <alexander@medvednikov.com>
Date: Fri, 27 Sep 2019 04:00:48 +0300
Subject: [PATCH] bring back "parser: cache tokens (first step)"

This reverts commit 0fcdd7db358bf5580ca7a073faf7c2d109654f60.
---
 compiler/comptime.v |  4 +-
 compiler/main.v     |  2 +-
 compiler/parser.v   | 94 +++++++++++++++++++++++++++++++++++++++------
 compiler/scanner.v  | 20 ----------
 compiler/vfmt.v     |  2 +
 5 files changed, 88 insertions(+), 34 deletions(-)

diff --git a/compiler/comptime.v b/compiler/comptime.v
index 70788f8930..e6b9fad910 100644
--- a/compiler/comptime.v
+++ b/compiler/comptime.v
@@ -20,7 +20,7 @@ fn (p mut Parser) comp_time() {
 		}
 		name := p.check_name()
 		p.fspace()
-		if name in SupportedPlatforms {
+		if name in supported_platforms {
 			ifdef_name := os_name_to_ifdef(name)
 			if not {
 				p.genln('#ifndef $ifdef_name')
@@ -42,7 +42,7 @@ fn (p mut Parser) comp_time() {
 		}
 		else {
 			println('Supported platforms:')
-			println(SupportedPlatforms)
+			println(supported_platforms)
 			p.error('unknown platform `$name`')
 		}
 		if_returns := p.returns
diff --git a/compiler/main.v b/compiler/main.v
index c4571a6eb8..952a82b57a 100644
--- a/compiler/main.v
+++ b/compiler/main.v
@@ -27,7 +27,7 @@ enum BuildMode {
 }
 
 const (
-	SupportedPlatforms = ['windows', 'mac', 'linux', 'freebsd', 'openbsd',
+	supported_platforms = ['windows', 'mac', 'linux', 'freebsd', 'openbsd',
 		'netbsd', 'dragonfly', 'msvc', 'android', 'js', 'solaris']
 	ModPath            = os.home_dir() + '/.vmodules/'
 )
diff --git a/compiler/parser.v b/compiler/parser.v
index 009b454698..da61c9c93b 100644
--- a/compiler/parser.v
+++ b/compiler/parser.v
@@ -9,6 +9,15 @@ import (
 	strings
 )
 
+// TODO rename to Token
+// TODO rename enum Token to TokenType
+struct Tok {
+       tok Token
+       lit string
+       line_nr int
+//       col int
+}
+
 struct Parser {
 	file_path      string // "/home/user/hello.v"
 	file_name      string // "hello.v"
@@ -21,7 +30,7 @@ struct Parser {
 	pref           &Preferences // Preferences shared from V struct
 mut:
 	scanner        &Scanner
-	// tokens         []Token // TODO cache all tokens, right now they have to be scanned twice
+	tokens         []Tok
 	token_idx      int
 	tok            Token
 	prev_tok       Token
@@ -124,8 +133,20 @@ fn (v mut V) new_parser(path string) Parser {
 	v.cgen.line_directives = v.pref.is_debuggable
 	v.cgen.file = path
 
-	p.next()
-	// p.scanner.debug_tokens()
+	for {
+	       res := p.scanner.scan()
+	       p.tokens << Tok {
+	               tok: res.tok
+	               lit: res.lit
+	               line_nr: p.scanner.line_nr
+	       }
+	        if res.tok == .eof {
+	                break
+	        }
+	}
+
+	//p.next()
+	//p.scanner.debug_tokens()
 	return p
 }
 
@@ -136,6 +157,52 @@ fn (p mut Parser) set_current_fn(f Fn) {
 }
 
 fn (p mut Parser) next() {
+	 p.prev_tok2 = p.prev_tok
+	 p.prev_tok = p.tok
+	 p.scanner.prev_tok = p.tok
+	 if p.token_idx >= p.tokens.len {
+	         p.tok = Token.eof
+	         p.lit = ''
+	         return
+	 }
+	 res := p.tokens[p.token_idx]
+	 p.token_idx++
+	 p.tok = res.tok
+	 p.lit = res.lit
+	 p.scanner.line_nr = res.line_nr
+ }
+
+
+fn (p & Parser) peek() Token {
+	if p.token_idx >= p.tokens.len - 2 {
+		return Token.eof
+	}
+	tok := p.tokens[p.token_idx]
+	return tok.tok
+/*
+	// save scanner state
+	pos := s.pos
+	line := s.line_nr
+	inside_string := s.inside_string
+	inter_start := s.inter_start
+	inter_end := s.inter_end
+
+	res := s.scan()
+	tok := res.tok
+
+	// restore scanner state
+	s.pos = pos
+	s.line_nr = line
+	s.inside_string = inside_string
+	s.inter_start = inter_start
+	s.inter_end = inter_end
+	return tok
+	*/
+}
+
+
+
+fn (p mut Parser) next_old() {
 	p.prev_tok2 = p.prev_tok
 	p.prev_tok = p.tok
 	p.scanner.prev_tok = p.tok
@@ -155,6 +222,8 @@ fn (p &Parser) log(s string) {
 
 fn (p mut Parser) parse(pass Pass) {
 	p.pass = pass
+	p.token_idx = 0
+	p.next()
 	//p.log('\nparse() run=$p.pass file=$p.file_name tok=${p.strtok()}')// , "script_file=", script_file)
 	// `module main` is not required if it's a single file program
 	if p.is_script || p.pref.is_test {
@@ -1579,14 +1648,16 @@ fn (p mut Parser) name_expr() string {
 	}
 	// //////////////////////////
 	// module ?
-	// (Allow shadowing `gg = gg.newcontext(); gg.draw_triangle();` )
-	if p.peek() == .dot && ((name == p.mod && p.table.known_mod(name)) || p.import_table.known_alias(name))
-		&& !p.known_var(name) && !is_c {
+	if p.peek() == .dot && ((name == p.mod && p.table.known_mod(name)) ||
+		p.import_table.known_alias(name))	&& !is_c &&
+		!p.known_var(name)	// Allow shadowing (`gg = gg.newcontext(); gg.foo()`)
+	{
 		mut mod := name
 		// must be aliased module
 		if name != p.mod && p.import_table.known_alias(name) {
 			p.import_table.register_used_import(name)
-			// we replaced "." with "_dot_" in p.mod for C variable names, do same here.
+			// we replaced "." with "_dot_" in p.mod for C variable names,
+			// do same here.
 			mod = p.import_table.resolve_alias(name).replace('.', '_dot_')
 		}
 		p.next()
@@ -1596,7 +1667,8 @@ fn (p mut Parser) name_expr() string {
 		name = prepend_mod(mod, name)
 	}
 	else if !p.table.known_type(name) && !p.known_var(name) &&
-	!p.table.known_fn(name) && !p.table.known_const(name) && !is_c {
+		!p.table.known_fn(name) && !p.table.known_const(name) && !is_c
+	{
 		name = p.prepend_mod(name)
 	}
 	// Variable
@@ -2800,10 +2872,10 @@ fn (p mut Parser) array_init() string {
 			typ = val_typ
 			// fixed width array initialization? (`arr := [20]byte`)
 			if is_integer && p.tok == .rsbr && p.peek() == .name {
-				nextc := p.scanner.text[p.scanner.pos + 1]
+				//nextc := p.scanner.text[p.scanner.pos + 1]
 				// TODO whitespace hack
 				// Make sure there's no space in `[10]byte`
-				if !nextc.is_space() {
+				//if !nextc.is_space() {
 					p.check(.rsbr)
 					array_elem_typ := p.get_type()
 					if !p.table.known_type(array_elem_typ) {
@@ -2816,7 +2888,7 @@ fn (p mut Parser) array_init() string {
 						return '[${p.mod}__$lit]$array_elem_typ'
 					}
 					return '[$lit]$array_elem_typ'
-				}
+				//}
 			}
 		}
 		if val_typ != typ {
diff --git a/compiler/scanner.v b/compiler/scanner.v
index bbc4c0e41b..a790d2409d 100644
--- a/compiler/scanner.v
+++ b/compiler/scanner.v
@@ -785,26 +785,6 @@ fn (s mut Scanner) ident_char() string {
 	return if c == '\'' { '\\' + c } else { c }
 }
 
-fn (s mut Scanner) peek() Token {
-	// save scanner state
-	pos := s.pos
-	line := s.line_nr
-	inside_string := s.inside_string
-	inter_start := s.inter_start
-	inter_end := s.inter_end
-
-	res := s.scan()
-	tok := res.tok
-
-	// restore scanner state
-	s.pos = pos
-	s.line_nr = line
-	s.inside_string = inside_string
-	s.inter_start = inter_start
-	s.inter_end = inter_end
-	return tok
-}
-
 fn (s &Scanner) expect(want string, start_pos int) bool {
 	end_pos := start_pos + want.len
 	if start_pos < 0 || start_pos >= s.text.len {
diff --git a/compiler/vfmt.v b/compiler/vfmt.v
index 74db232ac6..69a5b05e99 100644
--- a/compiler/vfmt.v
+++ b/compiler/vfmt.v
@@ -37,6 +37,7 @@ fn (p mut Parser) fgenln(s string) {
 	p.scanner.fgenln(s)
 }
 
+/*
 fn (p mut Parser) peek() Token {
 	for {
 		p.cgen.line = p.scanner.line_nr + 1
@@ -47,6 +48,7 @@ fn (p mut Parser) peek() Token {
 	}
 	return .eof // TODO can never get here - v doesn't know that
 }
+*/
 
 fn (p mut Parser) fmt_inc() {
 	p.scanner.fmt_indent++