From 666509623e77eebc2de9821d9b146e85d9ae61b5 Mon Sep 17 00:00:00 2001
From: joe-conigliaro <joe-conigliaro@users.noreply.github.com>
Date: Sun, 24 Nov 2019 03:31:28 +1100
Subject: [PATCH] compiler: use token.pos instead of token.col, calculate col
 when needed

---
 vlib/compiler/compile_errors.v | 45 +++++++++-------------------------
 vlib/compiler/parser.v         |  5 ++--
 vlib/compiler/token.v          |  2 +-
 3 files changed, 15 insertions(+), 37 deletions(-)

diff --git a/vlib/compiler/compile_errors.v b/vlib/compiler/compile_errors.v
index 9863f2d4bc..e4ed4fbb12 100644
--- a/vlib/compiler/compile_errors.v
+++ b/vlib/compiler/compile_errors.v
@@ -208,44 +208,21 @@ fn (s mut Scanner) goto_scanner_position(scp ScannerPos) {
 	s.last_nl_pos = scp.last_nl_pos
 }
 
-// get_scanner_pos_of_token rescans *the whole source* till it reaches {t.line_nr, t.col} .
-fn (s mut Scanner) get_scanner_pos_of_token(t &Token) ScannerPos {
-	// This rescanning is done just once on error, so it is fine for now.
-	// Be careful for the performance implications, if you want to
-	// do it more frequently. The alternative would be to store
-	// the scanpos (12 bytes) for each token, and there are potentially many tokens.
-	tline := t.line_nr
-	tcol  := if t.line_nr == 0 { t.col + 1 } else { t.col - 1 }
-	// save the current scanner position, it will be restored later
-	cpos := s.get_scanner_pos()
-	mut sptoken := ScannerPos{}
-	// Starting from the start, scan the source lines
-	// till the desired tline is reached, then
-	// s.pos + tcol would be the proper position
-	// of the token.
-	s.goto_scanner_position(ScannerPos{})
-
-	maxline := imin( s.nlines, tline + 2 * error_context_after)
-	for {
-		if s.pos >= s.text.len { break }		
-		if s.line_nr > maxline { break }
-		////////////////////////////////////////
-		if tline == s.line_nr {
-			sptoken = s.get_scanner_pos()
-			sptoken.pos += tcol
+fn (s mut Scanner) get_last_nl_from_pos(pos int) int {
+	for i := pos; i >= 0; i-- {
+		if s.text[i] == `\n` {
+			return i
 		}
-		s.ignore_line() s.eat_single_newline()
 	}
-	//////////////////////////////////////////////////
-	s.goto_scanner_position(cpos)
-	return sptoken
+	return 0
 }
 
-fn (s mut Scanner) eat_single_newline(){
-	if s.pos >= s.text.len { return }
-	if s.expect('\r\n', s.pos) { s.pos += 2 return }
-	if s.text[ s.pos ] == `\n` { s.pos ++ return }
-	if s.text[ s.pos ] == `\r` { s.pos ++ return }
+fn (s mut Scanner) get_scanner_pos_of_token(tok &Token) ScannerPos {
+	return ScannerPos{
+		pos: tok.pos
+		line_nr: tok.line_nr
+		last_nl_pos: s.get_last_nl_from_pos(tok.pos)
+	}
 }
 
 ///////////////////////////////
diff --git a/vlib/compiler/parser.v b/vlib/compiler/parser.v
index 6e725f6a65..c7039dc51f 100644
--- a/vlib/compiler/parser.v
+++ b/vlib/compiler/parser.v
@@ -188,7 +188,7 @@ fn (p mut Parser) scan_tokens() {
 			tok: res.tok
 			lit: res.lit
 			line_nr: p.scanner.line_nr
-			col: p.scanner.pos - p.scanner.last_nl_pos
+			pos: p.scanner.pos
 		}
 		if res.tok == .eof {
 				break
@@ -2384,7 +2384,8 @@ fn (p mut Parser) array_init() string {
 			if is_integer && p.tok == .rsbr && p.peek() == .name &&
 				p.cur_tok().line_nr == p.peek_token().line_nr {
 				// there is no space between `[10]` and `byte`
-				if p.cur_tok().col + p.peek_token().lit.len == p.peek_token().col {
+				// if p.cur_tok().col + p.peek_token().lit.len == p.peek_token().col {
+				if p.cur_tok().pos + p.peek_token().lit.len == p.peek_token().pos {
 					p.check(.rsbr)
 					array_elem_typ := p.get_type()
 					if !p.table.known_type(array_elem_typ) {
diff --git a/vlib/compiler/token.v b/vlib/compiler/token.v
index f285362837..cae409e24e 100644
--- a/vlib/compiler/token.v
+++ b/vlib/compiler/token.v
@@ -9,7 +9,7 @@ struct Token {
 	lit      string // literal representation of the token
 	line_nr  int // the line number in the source where the token occured
 	name_idx int // name table index for O(1) lookup
-	col      int // the column where the token ends
+	pos      int // the position of the token in scanner text
 }