scanner, token: add column information to tokens (#9407)

vlib/v/ast/ast.v

@@ -1440,6 +1440,7 @@ pub fn (expr Expr) position() token.Position {
 				line_nr: expr.pos.line_nr
 				pos: left_pos.pos
 				len: right_pos.pos - left_pos.pos + right_pos.len
+				col: left_pos.col
 				last_line: right_pos.last_line
 			}
 		}
@@ -1563,6 +1564,7 @@ pub fn (node Node) position() token.Position {
 						line_nr: -1
 						pos: -1
 						last_line: -1
+						col: -1
 					}
 				}
 			}

vlib/v/checker/checker.v

@@ -4402,8 +4402,7 @@ fn (mut c Checker) at_expr(mut node ast.AtExpr) table.Type {
 			node.val = (node.pos.line_nr + 1).str()
 		}
 		.column_nr {
-			_, column := util.filepath_pos_to_source_and_column(c.file.path, node.pos)
+			node.val = (node.pos.col + 1).str()
-			node.val = (column + 1).str()
 		}
 		.vhash {
 			node.val = util.vhash()

vlib/v/parser/if_match.v

@@ -257,6 +257,7 @@ fn (mut p Parser) match_expr() ast.MatchExpr {
 		line_nr: match_first_pos.line_nr
 		pos: match_first_pos.pos
 		len: match_last_pos.pos - match_first_pos.pos + match_last_pos.len
+		col: match_first_pos.col
 	}
 	if p.tok.kind == .rcbr {
 		p.check(.rcbr)
@@ -401,6 +402,7 @@ fn (mut p Parser) select_expr() ast.SelectExpr {
 			line_nr: branch_first_pos.line_nr
 			pos: branch_first_pos.pos
 			len: branch_last_pos.pos - branch_first_pos.pos + branch_last_pos.len
+			col: branch_first_pos.col
 		}
 		post_comments := p.eat_comments({})
 		pos.update_last_line(p.prev_tok.line_nr)
@@ -425,6 +427,7 @@ fn (mut p Parser) select_expr() ast.SelectExpr {
 		line_nr: match_first_pos.line_nr
 		pos: match_first_pos.pos
 		len: match_last_pos.pos - match_first_pos.pos + match_last_pos.len
+		col: match_first_pos.col
 	}
 	if p.tok.kind == .rcbr {
 		p.check(.rcbr)

vlib/v/parser/struct.v

@@ -389,6 +389,7 @@ fn (mut p Parser) struct_init(short_syntax bool) ast.StructInit {
 				line_nr: first_field_pos.line_nr
 				pos: first_field_pos.pos
 				len: field_len
+				col: first_field_pos.col
 			}
 		}
 		i++

vlib/v/scanner/scanner.v

@@ -26,7 +26,8 @@ pub mut:
 	text              string // the whole text of the file
 	pos               int    // current position in the file, first character is s.text[0]
 	line_nr           int    // current line number
-	last_nl_pos       int    // for calculating column
+	last_nl_pos       int = -1 // for calculating column
+	is_crlf           bool   // special check when computing columns
 	is_inside_string  bool   // set to true in a string, *at the start* of an $var or ${expr}
 	is_inter_start    bool   // for hacky string interpolation TODO simplify
 	is_inter_end      bool
@@ -177,6 +178,7 @@ fn (mut s Scanner) new_token(tok_kind token.Kind, lit string, len int) token.Tok
 		kind: tok_kind
 		lit: lit
 		line_nr: s.line_nr + line_offset
+		col: mu.max(1, s.current_column() - len + 1)
 		pos: s.pos - len + 1
 		len: len
 		tidx: cidx
@@ -189,6 +191,7 @@ fn (s &Scanner) new_eof_token() token.Token {
 		kind: .eof
 		lit: ''
 		line_nr: s.line_nr + 1
+		col: 1
 		pos: s.pos
 		len: 1
 		tidx: s.tidx
@@ -203,6 +206,7 @@ fn (mut s Scanner) new_multiline_token(tok_kind token.Kind, lit string, len int,
 		kind: tok_kind
 		lit: lit
 		line_nr: start_line + 1
+		col: mu.max(1, s.current_column() - len + 1)
 		pos: s.pos - len + 1
 		len: len
 		tidx: cidx
@@ -495,6 +499,9 @@ fn (mut s Scanner) skip_whitespace() {
 		if util.is_nl(s.text[s.pos]) && s.is_vh {
 			return
 		}
+		if s.pos + 1 < s.text.len && s.text[s.pos] == `\r` && s.text[s.pos + 1] == `\n` {
+			s.is_crlf = true
+		}
 		// Count \r\n as one line
 		if util.is_nl(s.text[s.pos]) && !s.expect('\r\n', s.pos - 1) {
 			s.inc_line_number()
@@ -863,19 +870,6 @@ fn (mut s Scanner) text_scan() token.Token {
 				}
 				return s.new_token(.name, name, name.len)
 			}
-			/*
-			case `\r`:
-		if nextc == `\n` {
-			s.pos++
-			s.last_nl_pos = s.pos
-			return s.new_token(.nl, '')
-		}
-	 }
-	case `\n`:
-		s.last_nl_pos = s.pos
-		return s.new_token(.nl, '')
-	 }
-			*/
 			`.` {
 				if nextc == `.` {
 					s.pos++
@@ -1298,7 +1292,10 @@ fn (mut s Scanner) eat_to_end_of_line() {
 
 [inline]
 fn (mut s Scanner) inc_line_number() {
-	s.last_nl_pos = s.pos
+	s.last_nl_pos = mu.min(s.text.len - 1, s.pos)
+	if s.is_crlf {
+		s.last_nl_pos++
+	}
 	s.line_nr++
 	s.line_ends << s.pos
 	if s.line_nr > s.nr_lines {
@@ -1331,6 +1328,7 @@ pub fn (mut s Scanner) warn(msg string) {
 	pos := token.Position{
 		line_nr: s.line_nr
 		pos: s.pos
+		col: s.current_column() - 1
 	}
 	if s.pref.output_mode == .stdout {
 		eprintln(util.formatted_error('warning:', msg, s.file_path, pos))
@@ -1348,6 +1346,7 @@ pub fn (mut s Scanner) error(msg string) {
 	pos := token.Position{
 		line_nr: s.line_nr
 		pos: s.pos
+		col: s.current_column() - 1
 	}
 	if s.pref.output_mode == .stdout {
 		eprintln(util.formatted_error('error:', msg, s.file_path, pos))
@@ -1371,6 +1370,7 @@ fn (mut s Scanner) vet_error(msg string, fix vet.FixKind) {
 		file_path: s.file_path
 		pos: token.Position{
 			line_nr: s.line_nr
+			col: s.current_column() - 1
 		}
 		kind: .error
 		fix: fix

vlib/v/token/position.v

@@ -8,12 +8,13 @@ pub:
 	len     int // length of the literal in the source
 	line_nr int // the line number in the source where the token occured
 	pos     int // the position of the token in scanner text
+	col     int // the column in the source where the token occured
 pub mut:
 	last_line int // the line number where the ast object ends (used by vfmt)
 }
 
 pub fn (pos Position) str() string {
-	return 'Position{ line_nr: $pos.line_nr, last_line: $pos.last_line, pos: $pos.pos, len: $pos.len }'
+	return 'Position{ line_nr: $pos.line_nr, last_line: $pos.last_line, pos: $pos.pos, col: $pos.col, len: $pos.len }'
 }
 
 pub fn (pos Position) extend(end Position) Position {
@@ -30,6 +31,7 @@ pub fn (pos Position) extend_with_last_line(end Position, last_line int) Positio
 		line_nr: pos.line_nr
 		last_line: last_line - 1
 		pos: pos.pos
+		col: pos.col
 	}
 }
 
@@ -44,5 +46,6 @@ pub fn (tok &Token) position() Position {
 		line_nr: tok.line_nr - 1
 		pos: tok.pos
 		last_line: tok.line_nr - 1
+		col: tok.col - 1
 	}
 }

vlib/v/token/token.v

@@ -8,6 +8,7 @@ pub:
 	kind    Kind   // the token number/enum; for quick comparisons
 	lit     string // literal representation of the token
 	line_nr int    // the line number in the source where the token occured
+	col     int    // the column in the source where the token occured
 	// name_idx int // name table index for O(1) lookup
 	pos  int // the position of the token in scanner text
 	len  int // length of the literal

vlib/v/util/errors.v

@@ -83,9 +83,9 @@ pub fn formatted_error(kind string, omsg string, filepath string, pos token.Posi
 		}
 	}
 	//
-	source, column := filepath_pos_to_source_and_column(filepath, pos)
+	source := read_file(filepath) or { '' }
-	position := '$path:${pos.line_nr + 1}:${mu.max(1, column + 1)}:'
+	position := '$path:${pos.line_nr + 1}:${mu.max(1, pos.col + 1)}:'
-	scontext := source_context(kind, source, column, pos).join('\n')
+	scontext := source_context(kind, source, pos).join('\n')
 	final_position := bold(position)
 	final_kind := bold(color(kind, kind))
 	final_msg := emsg
@@ -94,23 +94,7 @@ pub fn formatted_error(kind string, omsg string, filepath string, pos token.Posi
 	return '$final_position $final_kind $final_msg$final_context'.trim_space()
 }
 
-pub fn filepath_pos_to_source_and_column(filepath string, pos token.Position) (string, int) {
+pub fn source_context(kind string, source string, pos token.Position) []string {
-	// TODO: optimize this; may be use a cache.
-	// The column should not be so computationally hard to get.
-	source := read_file(filepath) or { '' }
-	mut p := mu.max(0, mu.min(source.len - 1, pos.pos))
-	if source.len > 0 {
-		for ; p >= 0; p-- {
-			if source[p] == `\n` || source[p] == `\r` {
-				break
-			}
-		}
-	}
-	column := mu.max(0, pos.pos - p - 1)
-	return source, column
-}
-
-pub fn source_context(kind string, source string, column int, pos token.Position) []string {
 	mut clines := []string{}
 	if source.len == 0 {
 		return clines
@@ -121,8 +105,8 @@ pub fn source_context(kind string, source string, column int, pos token.Position
 	tab_spaces := '    '
 	for iline := bline; iline <= aline; iline++ {
 		sline := source_lines[iline]
-		start_column := mu.max(0, mu.min(column, sline.len))
+		start_column := mu.max(0, mu.min(pos.col, sline.len))
-		end_column := mu.max(0, mu.min(column + mu.max(0, pos.len), sline.len))
+		end_column := mu.max(0, mu.min(pos.col + mu.max(0, pos.len), sline.len))
 		cline := if iline == pos.line_nr {
 			sline[..start_column] + color(kind, sline[start_column..end_column]) +
 				sline[end_column..]