2021-01-18 13:20:06 +01:00
|
|
|
|
// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved.
|
2019-12-22 02:34:37 +01:00
|
|
|
|
// Use of this source code is governed by an MIT license
|
|
|
|
|
// that can be found in the LICENSE file.
|
|
|
|
|
module scanner
|
|
|
|
|
|
2021-03-12 10:28:04 +01:00
|
|
|
|
import math.mathutil as mu
|
2020-04-26 07:35:59 +02:00
|
|
|
|
import os
|
2021-03-14 11:09:17 +01:00
|
|
|
|
import strconv
|
2020-04-26 07:35:59 +02:00
|
|
|
|
import v.token
|
|
|
|
|
import v.pref
|
|
|
|
|
import v.util
|
2021-01-09 15:11:49 +01:00
|
|
|
|
import v.vet
|
2020-11-20 10:44:19 +01:00
|
|
|
|
import v.errors
|
2019-12-22 02:34:37 +01:00
|
|
|
|
|
|
|
|
|
const (
|
|
|
|
|
single_quote = `\'`
|
|
|
|
|
double_quote = `"`
|
2020-07-04 15:14:30 +02:00
|
|
|
|
// char used as number separator
|
|
|
|
|
num_sep = `_`
|
2021-05-03 13:14:32 +02:00
|
|
|
|
b_lf = 10
|
|
|
|
|
b_cr = 13
|
2019-12-22 02:34:37 +01:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
pub struct Scanner {
|
2020-05-09 15:16:48 +02:00
|
|
|
|
pub mut:
|
2021-01-28 14:21:19 +01:00
|
|
|
|
file_path string // '/path/to/file.v'
|
|
|
|
|
file_base string // 'file.v'
|
|
|
|
|
text string // the whole text of the file
|
|
|
|
|
pos int // current position in the file, first character is s.text[0]
|
|
|
|
|
line_nr int // current line number
|
2021-03-23 06:23:46 +01:00
|
|
|
|
last_nl_pos int = -1 // for calculating column
|
|
|
|
|
is_crlf bool // special check when computing columns
|
2021-01-28 14:21:19 +01:00
|
|
|
|
is_inside_string bool // set to true in a string, *at the start* of an $var or ${expr}
|
|
|
|
|
is_inter_start bool // for hacky string interpolation TODO simplify
|
2021-01-12 04:38:43 +01:00
|
|
|
|
is_inter_end bool
|
|
|
|
|
is_enclosed_inter bool
|
|
|
|
|
line_comment string
|
2019-12-28 09:43:22 +01:00
|
|
|
|
// prev_tok TokenKind
|
2020-04-02 12:23:18 +02:00
|
|
|
|
is_started bool
|
|
|
|
|
is_print_line_on_error bool
|
|
|
|
|
is_print_colored_error bool
|
|
|
|
|
is_print_rel_paths_on_error bool
|
|
|
|
|
quote byte // which quote is used to denote current string: ' or "
|
2020-12-09 19:58:27 +01:00
|
|
|
|
inter_quote byte
|
2020-04-02 12:23:18 +02:00
|
|
|
|
line_ends []int // the positions of source lines ends (i.e. \n signs)
|
2021-01-03 21:10:25 +01:00
|
|
|
|
nr_lines int // total number of lines in the source file that were scanned
|
|
|
|
|
is_vh bool // Keep newlines
|
|
|
|
|
is_fmt bool // Used for v fmt.
|
2020-04-02 12:23:18 +02:00
|
|
|
|
comments_mode CommentsMode
|
2020-09-22 05:28:29 +02:00
|
|
|
|
is_inside_toplvl_statement bool // *only* used in comments_mode: .toplevel_comments, toggled by parser
|
2020-06-06 17:47:16 +02:00
|
|
|
|
all_tokens []token.Token // *only* used in comments_mode: .toplevel_comments, contains all tokens
|
|
|
|
|
tidx int
|
2020-05-28 18:22:11 +02:00
|
|
|
|
eofs int
|
2020-07-11 11:41:39 +02:00
|
|
|
|
pref &pref.Preferences
|
2020-11-20 10:44:19 +01:00
|
|
|
|
errors []errors.Error
|
|
|
|
|
warnings []errors.Warning
|
2021-03-22 18:43:06 +01:00
|
|
|
|
notices []errors.Notice
|
2021-01-09 15:11:49 +01:00
|
|
|
|
vet_errors []vet.Error
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
|
2020-06-06 17:47:16 +02:00
|
|
|
|
/*
|
|
|
|
|
How the .toplevel_comments mode works:
|
2020-02-29 17:51:35 +01:00
|
|
|
|
|
2020-06-06 17:47:16 +02:00
|
|
|
|
In this mode, the scanner scans *everything* at once, before parsing starts,
|
|
|
|
|
including all the comments, and stores the results in an buffer s.all_tokens.
|
|
|
|
|
|
|
|
|
|
Then .scan() just returns s.all_tokens[ s.tidx++ ] *ignoring* the
|
|
|
|
|
comment tokens. In other words, by default in this mode, the parser
|
|
|
|
|
*will not see any comments* inside top level statements, so it has
|
|
|
|
|
no reason to complain about them.
|
|
|
|
|
|
|
|
|
|
When the parser determines, that it is outside of a top level statement,
|
|
|
|
|
it tells the scanner to backtrack s.tidx to the current p.tok index,
|
|
|
|
|
then it changes .is_inside_toplvl_statement to false , and refills its
|
2021-02-24 19:03:53 +01:00
|
|
|
|
lookahead buffer (i.e. p.peek_tok), from the scanner.
|
2020-06-06 17:47:16 +02:00
|
|
|
|
|
|
|
|
|
In effect, from the parser's point of view, the next tokens, that it will
|
|
|
|
|
receive with p.next(), will be the same, as if comments are not ignored
|
|
|
|
|
anymore, *between* top level statements.
|
|
|
|
|
|
|
|
|
|
When the parser determines, that it is going again inside a top level
|
|
|
|
|
statement, it does the same, this time setting .is_inside_toplvl_statement
|
|
|
|
|
to true, again refilling the lookahead buffer => calling .next() in this
|
|
|
|
|
mode, will again ignore all the comment tokens, till the top level statement
|
|
|
|
|
is finished.
|
|
|
|
|
*/
|
|
|
|
|
// The different kinds of scanner modes:
|
|
|
|
|
//
|
|
|
|
|
// .skip_comments - simplest/fastest, just ignores all comments early.
|
|
|
|
|
// This mode is used by the compiler itself.
|
|
|
|
|
//
|
|
|
|
|
// .parse_comments is used by vfmt. Ideally it should handle inline /* */
|
|
|
|
|
// comments too, i.e. it returns every kind of comment as a new token.
|
|
|
|
|
//
|
|
|
|
|
// .toplevel_comments is used by vdoc, parses *only* top level ones
|
|
|
|
|
// that are *outside* structs/enums/fns.
|
2020-02-29 17:51:35 +01:00
|
|
|
|
pub enum CommentsMode {
|
|
|
|
|
skip_comments
|
|
|
|
|
parse_comments
|
2020-06-06 17:47:16 +02:00
|
|
|
|
toplevel_comments
|
2020-02-29 17:51:35 +01:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-22 02:34:37 +01:00
|
|
|
|
// new scanner from file.
|
2020-07-11 11:41:39 +02:00
|
|
|
|
pub fn new_scanner_file(file_path string, comments_mode CommentsMode, pref &pref.Preferences) &Scanner {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
if !os.exists(file_path) {
|
|
|
|
|
verror("$file_path doesn't exist")
|
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
raw_text := util.read_file(file_path) or {
|
2020-04-06 18:39:58 +02:00
|
|
|
|
verror(err)
|
2020-05-27 05:42:48 +02:00
|
|
|
|
return voidptr(0)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2021-02-24 19:03:53 +01:00
|
|
|
|
mut s := &Scanner{
|
|
|
|
|
pref: pref
|
|
|
|
|
text: raw_text
|
|
|
|
|
is_print_line_on_error: true
|
|
|
|
|
is_print_colored_error: true
|
|
|
|
|
is_print_rel_paths_on_error: true
|
|
|
|
|
is_fmt: pref.is_fmt
|
|
|
|
|
comments_mode: comments_mode
|
|
|
|
|
file_path: file_path
|
|
|
|
|
file_base: os.base(file_path)
|
|
|
|
|
}
|
|
|
|
|
s.init_scanner()
|
2019-12-22 02:34:37 +01:00
|
|
|
|
return s
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// new scanner from string.
|
2020-07-11 11:41:39 +02:00
|
|
|
|
pub fn new_scanner(text string, comments_mode CommentsMode, pref &pref.Preferences) &Scanner {
|
2021-02-24 19:03:53 +01:00
|
|
|
|
mut s := &Scanner{
|
2020-07-11 11:41:39 +02:00
|
|
|
|
pref: pref
|
2019-12-22 02:34:37 +01:00
|
|
|
|
text: text
|
2020-04-02 12:23:18 +02:00
|
|
|
|
is_print_line_on_error: true
|
|
|
|
|
is_print_colored_error: true
|
|
|
|
|
is_print_rel_paths_on_error: true
|
2020-12-09 02:09:46 +01:00
|
|
|
|
is_fmt: pref.is_fmt
|
2020-02-29 17:51:35 +01:00
|
|
|
|
comments_mode: comments_mode
|
2020-12-09 02:09:46 +01:00
|
|
|
|
file_path: 'internal_memory'
|
2021-01-28 14:21:19 +01:00
|
|
|
|
file_base: 'internal_memory'
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2021-02-24 19:03:53 +01:00
|
|
|
|
s.init_scanner()
|
|
|
|
|
return s
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn (mut s Scanner) init_scanner() {
|
|
|
|
|
util.get_timers().measure_pause('PARSE')
|
|
|
|
|
s.scan_all_tokens_in_buffer(s.comments_mode)
|
|
|
|
|
util.get_timers().measure_resume('PARSE')
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
[unsafe]
|
|
|
|
|
pub fn (mut s Scanner) free() {
|
|
|
|
|
unsafe {
|
|
|
|
|
s.text.free()
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
|
2020-06-06 17:47:16 +02:00
|
|
|
|
[inline]
|
|
|
|
|
fn (s &Scanner) should_parse_comment() bool {
|
2021-01-23 09:33:22 +01:00
|
|
|
|
return (s.comments_mode == .parse_comments)
|
|
|
|
|
|| (s.comments_mode == .toplevel_comments && !s.is_inside_toplvl_statement)
|
2020-06-06 17:47:16 +02:00
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
|
2020-06-06 17:47:16 +02:00
|
|
|
|
// NB: this is called by v's parser
|
|
|
|
|
pub fn (mut s Scanner) set_is_inside_toplevel_statement(newstate bool) {
|
|
|
|
|
s.is_inside_toplvl_statement = newstate
|
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
|
2020-06-06 17:47:16 +02:00
|
|
|
|
pub fn (mut s Scanner) set_current_tidx(cidx int) {
|
|
|
|
|
mut tidx := if cidx < 0 { 0 } else { cidx }
|
|
|
|
|
tidx = if tidx > s.all_tokens.len { s.all_tokens.len } else { tidx }
|
|
|
|
|
s.tidx = tidx
|
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
|
2020-07-11 20:27:39 +02:00
|
|
|
|
[inline]
|
2020-06-06 17:47:16 +02:00
|
|
|
|
fn (mut s Scanner) new_token(tok_kind token.Kind, lit string, len int) token.Token {
|
|
|
|
|
cidx := s.tidx
|
|
|
|
|
s.tidx++
|
2021-01-29 11:17:59 +01:00
|
|
|
|
line_offset := if tok_kind == .hash { 0 } else { 1 }
|
2019-12-28 09:15:32 +01:00
|
|
|
|
return token.Token{
|
2020-02-12 17:39:35 +01:00
|
|
|
|
kind: tok_kind
|
|
|
|
|
lit: lit
|
2021-01-29 11:17:59 +01:00
|
|
|
|
line_nr: s.line_nr + line_offset
|
2021-03-23 06:23:46 +01:00
|
|
|
|
col: mu.max(1, s.current_column() - len + 1)
|
2020-04-10 00:09:34 +02:00
|
|
|
|
pos: s.pos - len + 1
|
|
|
|
|
len: len
|
2020-06-06 17:47:16 +02:00
|
|
|
|
tidx: cidx
|
2020-02-12 17:39:35 +01:00
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
|
2021-02-24 19:03:53 +01:00
|
|
|
|
[inline]
|
|
|
|
|
fn (s &Scanner) new_eof_token() token.Token {
|
|
|
|
|
return token.Token{
|
|
|
|
|
kind: .eof
|
|
|
|
|
lit: ''
|
|
|
|
|
line_nr: s.line_nr + 1
|
2021-03-23 08:51:09 +01:00
|
|
|
|
col: s.current_column()
|
2021-02-24 19:03:53 +01:00
|
|
|
|
pos: s.pos
|
|
|
|
|
len: 1
|
|
|
|
|
tidx: s.tidx
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-01-19 14:49:40 +01:00
|
|
|
|
[inline]
|
2021-01-27 13:53:41 +01:00
|
|
|
|
fn (mut s Scanner) new_multiline_token(tok_kind token.Kind, lit string, len int, start_line int) token.Token {
|
2021-01-19 14:49:40 +01:00
|
|
|
|
cidx := s.tidx
|
|
|
|
|
s.tidx++
|
|
|
|
|
return token.Token{
|
|
|
|
|
kind: tok_kind
|
|
|
|
|
lit: lit
|
|
|
|
|
line_nr: start_line + 1
|
2021-03-23 06:23:46 +01:00
|
|
|
|
col: mu.max(1, s.current_column() - len + 1)
|
2021-01-19 14:49:40 +01:00
|
|
|
|
pos: s.pos - len + 1
|
|
|
|
|
len: len
|
|
|
|
|
tidx: cidx
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-05-05 22:45:23 +02:00
|
|
|
|
[direct_array_access; inline]
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_name() string {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
start := s.pos
|
2020-02-25 22:58:51 +01:00
|
|
|
|
s.pos++
|
2021-05-05 22:45:23 +02:00
|
|
|
|
for s.pos < s.text.len {
|
|
|
|
|
c := s.text[s.pos]
|
|
|
|
|
if !(util.is_name_char(c) || c.is_digit()) {
|
|
|
|
|
break
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos++
|
|
|
|
|
}
|
|
|
|
|
name := s.text[start..s.pos]
|
|
|
|
|
s.pos--
|
|
|
|
|
return name
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-23 13:48:43 +01:00
|
|
|
|
fn (s Scanner) num_lit(start int, end int) string {
|
|
|
|
|
if s.is_fmt {
|
|
|
|
|
return s.text[start..end]
|
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
unsafe {
|
2020-12-23 13:48:43 +01:00
|
|
|
|
txt := s.text.str
|
2020-02-07 22:10:48 +01:00
|
|
|
|
mut b := malloc(end - start + 1) // add a byte for the endstring 0
|
|
|
|
|
mut i1 := 0
|
2020-07-04 15:14:30 +02:00
|
|
|
|
for i := start; i < end; i++ {
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if txt[i] != scanner.num_sep {
|
2020-02-07 22:10:48 +01:00
|
|
|
|
b[i1] = txt[i]
|
|
|
|
|
i1++
|
|
|
|
|
}
|
2020-01-23 03:28:25 +01:00
|
|
|
|
}
|
2020-02-07 22:10:48 +01:00
|
|
|
|
b[i1] = 0 // C string compatibility
|
2020-08-10 18:05:26 +02:00
|
|
|
|
return b.vstring_with_len(i1)
|
2020-01-23 03:28:25 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_bin_number() string {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
mut has_wrong_digit := false
|
2020-05-21 18:43:57 +02:00
|
|
|
|
mut first_wrong_digit_pos := 0
|
2020-02-25 11:11:12 +01:00
|
|
|
|
mut first_wrong_digit := `\0`
|
2020-01-23 03:28:25 +01:00
|
|
|
|
start_pos := s.pos
|
|
|
|
|
s.pos += 2 // skip '0b'
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if s.pos < s.text.len && s.text[s.pos] == scanner.num_sep {
|
2020-07-16 16:29:07 +02:00
|
|
|
|
s.error('separator `_` is only valid between digits in a numeric literal')
|
|
|
|
|
}
|
2020-02-25 22:58:51 +01:00
|
|
|
|
for s.pos < s.text.len {
|
2020-01-23 03:28:25 +01:00
|
|
|
|
c := s.text[s.pos]
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if c == scanner.num_sep && s.text[s.pos - 1] == scanner.num_sep {
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` consecutively')
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if !c.is_bin_digit() && c != scanner.num_sep {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
if (!c.is_digit() && !c.is_letter()) || s.is_inside_string {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
break
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if !has_wrong_digit {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
has_wrong_digit = true
|
2020-05-21 18:43:57 +02:00
|
|
|
|
first_wrong_digit_pos = s.pos
|
2020-02-25 11:11:12 +01:00
|
|
|
|
first_wrong_digit = c
|
|
|
|
|
}
|
2020-01-23 03:28:25 +01:00
|
|
|
|
}
|
|
|
|
|
s.pos++
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if s.text[s.pos - 1] == scanner.num_sep {
|
2021-01-22 23:24:48 +01:00
|
|
|
|
s.pos--
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` at the end of a numeric literal')
|
2020-09-22 05:28:29 +02:00
|
|
|
|
} else if start_pos + 2 == s.pos {
|
2020-05-21 15:20:36 +02:00
|
|
|
|
s.pos-- // adjust error position
|
2020-02-24 18:02:36 +01:00
|
|
|
|
s.error('number part of this binary is not provided')
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if has_wrong_digit {
|
2020-05-21 18:43:57 +02:00
|
|
|
|
s.pos = first_wrong_digit_pos // adjust error position
|
2020-07-04 15:14:30 +02:00
|
|
|
|
s.error('this binary number has unsuitable digit `$first_wrong_digit.str()`')
|
2020-02-25 11:11:12 +01:00
|
|
|
|
}
|
2020-12-23 13:48:43 +01:00
|
|
|
|
number := s.num_lit(start_pos, s.pos)
|
2020-01-23 03:28:25 +01:00
|
|
|
|
s.pos--
|
|
|
|
|
return number
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_hex_number() string {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
mut has_wrong_digit := false
|
2020-05-21 18:43:57 +02:00
|
|
|
|
mut first_wrong_digit_pos := 0
|
2020-02-25 11:11:12 +01:00
|
|
|
|
mut first_wrong_digit := `\0`
|
2019-12-22 02:34:37 +01:00
|
|
|
|
start_pos := s.pos
|
2020-12-12 14:20:38 +01:00
|
|
|
|
if s.pos + 2 >= s.text.len {
|
|
|
|
|
return '0x'
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos += 2 // skip '0x'
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if s.pos < s.text.len && s.text[s.pos] == scanner.num_sep {
|
2020-07-16 16:29:07 +02:00
|
|
|
|
s.error('separator `_` is only valid between digits in a numeric literal')
|
|
|
|
|
}
|
2020-02-25 22:58:51 +01:00
|
|
|
|
for s.pos < s.text.len {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
c := s.text[s.pos]
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if c == scanner.num_sep && s.text[s.pos - 1] == scanner.num_sep {
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` consecutively')
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if !c.is_hex_digit() && c != scanner.num_sep {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
if !c.is_letter() || s.is_inside_string {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
break
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if !has_wrong_digit {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
has_wrong_digit = true
|
2020-05-21 18:43:57 +02:00
|
|
|
|
first_wrong_digit_pos = s.pos
|
2020-02-25 11:11:12 +01:00
|
|
|
|
first_wrong_digit = c
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
s.pos++
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if s.text[s.pos - 1] == scanner.num_sep {
|
2021-01-22 23:24:48 +01:00
|
|
|
|
s.pos--
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` at the end of a numeric literal')
|
2020-09-22 05:28:29 +02:00
|
|
|
|
} else if start_pos + 2 == s.pos {
|
2020-05-21 15:20:36 +02:00
|
|
|
|
s.pos-- // adjust error position
|
2020-02-24 18:02:36 +01:00
|
|
|
|
s.error('number part of this hexadecimal is not provided')
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if has_wrong_digit {
|
2020-05-21 18:43:57 +02:00
|
|
|
|
s.pos = first_wrong_digit_pos // adjust error position
|
2020-07-04 15:14:30 +02:00
|
|
|
|
s.error('this hexadecimal number has unsuitable digit `$first_wrong_digit.str()`')
|
2020-02-25 11:11:12 +01:00
|
|
|
|
}
|
2020-12-23 13:48:43 +01:00
|
|
|
|
number := s.num_lit(start_pos, s.pos)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos--
|
|
|
|
|
return number
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_oct_number() string {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
mut has_wrong_digit := false
|
2020-05-21 18:43:57 +02:00
|
|
|
|
mut first_wrong_digit_pos := 0
|
2020-02-25 11:11:12 +01:00
|
|
|
|
mut first_wrong_digit := `\0`
|
2019-12-22 02:34:37 +01:00
|
|
|
|
start_pos := s.pos
|
2020-02-23 12:33:07 +01:00
|
|
|
|
s.pos += 2 // skip '0o'
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if s.pos < s.text.len && s.text[s.pos] == scanner.num_sep {
|
2020-07-16 16:29:07 +02:00
|
|
|
|
s.error('separator `_` is only valid between digits in a numeric literal')
|
|
|
|
|
}
|
2020-02-25 22:58:51 +01:00
|
|
|
|
for s.pos < s.text.len {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
c := s.text[s.pos]
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if c == scanner.num_sep && s.text[s.pos - 1] == scanner.num_sep {
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` consecutively')
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if !c.is_oct_digit() && c != scanner.num_sep {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
if (!c.is_digit() && !c.is_letter()) || s.is_inside_string {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
break
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if !has_wrong_digit {
|
2020-02-25 11:11:12 +01:00
|
|
|
|
has_wrong_digit = true
|
2020-05-21 18:43:57 +02:00
|
|
|
|
first_wrong_digit_pos = s.pos
|
2020-02-25 11:11:12 +01:00
|
|
|
|
first_wrong_digit = c
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
s.pos++
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if s.text[s.pos - 1] == scanner.num_sep {
|
2021-01-22 23:24:48 +01:00
|
|
|
|
s.pos--
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` at the end of a numeric literal')
|
2020-09-22 05:28:29 +02:00
|
|
|
|
} else if start_pos + 2 == s.pos {
|
2020-05-21 15:20:36 +02:00
|
|
|
|
s.pos-- // adjust error position
|
2020-02-24 18:02:36 +01:00
|
|
|
|
s.error('number part of this octal is not provided')
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if has_wrong_digit {
|
2020-05-21 18:43:57 +02:00
|
|
|
|
s.pos = first_wrong_digit_pos // adjust error position
|
2020-07-04 15:14:30 +02:00
|
|
|
|
s.error('this octal number has unsuitable digit `$first_wrong_digit.str()`')
|
2020-02-25 11:11:12 +01:00
|
|
|
|
}
|
2020-12-23 13:48:43 +01:00
|
|
|
|
number := s.num_lit(start_pos, s.pos)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos--
|
|
|
|
|
return number
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_dec_number() string {
|
2020-02-25 14:33:29 +01:00
|
|
|
|
mut has_wrong_digit := false
|
2020-05-21 18:43:57 +02:00
|
|
|
|
mut first_wrong_digit_pos := 0
|
2020-02-25 14:33:29 +01:00
|
|
|
|
mut first_wrong_digit := `\0`
|
2019-12-22 02:34:37 +01:00
|
|
|
|
start_pos := s.pos
|
|
|
|
|
// scan integer part
|
2020-02-25 14:33:29 +01:00
|
|
|
|
for s.pos < s.text.len {
|
2020-02-25 22:58:51 +01:00
|
|
|
|
c := s.text[s.pos]
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if c == scanner.num_sep && s.text[s.pos - 1] == scanner.num_sep {
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` consecutively')
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if !c.is_digit() && c != scanner.num_sep {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
if !c.is_letter() || c in [`e`, `E`] || s.is_inside_string {
|
2020-02-25 14:33:29 +01:00
|
|
|
|
break
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if !has_wrong_digit {
|
2020-02-25 14:33:29 +01:00
|
|
|
|
has_wrong_digit = true
|
2020-05-21 18:43:57 +02:00
|
|
|
|
first_wrong_digit_pos = s.pos
|
2020-02-25 22:58:51 +01:00
|
|
|
|
first_wrong_digit = c
|
2020-02-25 14:33:29 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos++
|
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
if s.text[s.pos - 1] == scanner.num_sep {
|
2021-01-22 23:24:48 +01:00
|
|
|
|
s.pos--
|
2020-07-28 08:09:19 +02:00
|
|
|
|
s.error('cannot use `_` at the end of a numeric literal')
|
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
mut call_method := false // true for, e.g., 5.str(), 5.5.str(), 5e5.str()
|
|
|
|
|
mut is_range := false // true for, e.g., 5..10
|
2019-12-22 02:34:37 +01:00
|
|
|
|
// scan fractional part
|
|
|
|
|
if s.pos < s.text.len && s.text[s.pos] == `.` {
|
|
|
|
|
s.pos++
|
2020-03-31 12:47:32 +02:00
|
|
|
|
if s.pos < s.text.len {
|
2020-04-18 17:49:27 +02:00
|
|
|
|
// 5.5, 5.5.str()
|
2020-03-31 12:47:32 +02:00
|
|
|
|
if s.text[s.pos].is_digit() {
|
|
|
|
|
for s.pos < s.text.len {
|
|
|
|
|
c := s.text[s.pos]
|
|
|
|
|
if !c.is_digit() {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
if !c.is_letter() || c in [`e`, `E`] || s.is_inside_string {
|
2020-04-18 17:49:27 +02:00
|
|
|
|
// 5.5.str()
|
|
|
|
|
if c == `.` && s.pos + 1 < s.text.len && s.text[s.pos + 1].is_letter() {
|
2020-03-31 12:47:32 +02:00
|
|
|
|
call_method = true
|
|
|
|
|
}
|
|
|
|
|
break
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if !has_wrong_digit {
|
2020-03-31 12:47:32 +02:00
|
|
|
|
has_wrong_digit = true
|
2020-05-21 18:43:57 +02:00
|
|
|
|
first_wrong_digit_pos = s.pos
|
2020-03-31 12:47:32 +02:00
|
|
|
|
first_wrong_digit = c
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
s.pos++
|
2020-02-25 14:33:29 +01:00
|
|
|
|
}
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if s.text[s.pos] == `.` {
|
|
|
|
|
// 5.. (a range)
|
2020-04-18 17:49:27 +02:00
|
|
|
|
is_range = true
|
|
|
|
|
s.pos--
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if s.text[s.pos] in [`e`, `E`] {
|
|
|
|
|
// 5.e5
|
|
|
|
|
} else if s.text[s.pos].is_letter() {
|
|
|
|
|
// 5.str()
|
2020-03-31 21:03:21 +02:00
|
|
|
|
call_method = true
|
2020-03-31 12:47:32 +02:00
|
|
|
|
s.pos--
|
2020-08-05 05:11:29 +02:00
|
|
|
|
} else {
|
2020-07-04 15:14:30 +02:00
|
|
|
|
// 5.
|
2020-04-18 17:49:27 +02:00
|
|
|
|
}
|
2020-03-31 21:03:21 +02:00
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
// scan exponential part
|
2020-04-18 17:49:27 +02:00
|
|
|
|
mut has_exp := false
|
|
|
|
|
if s.pos < s.text.len && s.text[s.pos] in [`e`, `E`] {
|
|
|
|
|
has_exp = true
|
2020-02-25 11:10:05 +01:00
|
|
|
|
s.pos++
|
|
|
|
|
if s.pos < s.text.len && s.text[s.pos] in [`-`, `+`] {
|
2020-02-17 02:35:01 +01:00
|
|
|
|
s.pos++
|
|
|
|
|
}
|
2020-02-25 14:33:29 +01:00
|
|
|
|
for s.pos < s.text.len {
|
2020-02-25 22:58:51 +01:00
|
|
|
|
c := s.text[s.pos]
|
|
|
|
|
if !c.is_digit() {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
if !c.is_letter() || s.is_inside_string {
|
2020-04-18 17:49:27 +02:00
|
|
|
|
// 5e5.str()
|
|
|
|
|
if c == `.` && s.pos + 1 < s.text.len && s.text[s.pos + 1].is_letter() {
|
2020-03-31 12:47:32 +02:00
|
|
|
|
call_method = true
|
|
|
|
|
}
|
2020-02-25 14:33:29 +01:00
|
|
|
|
break
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if !has_wrong_digit {
|
2020-02-25 14:33:29 +01:00
|
|
|
|
has_wrong_digit = true
|
2020-05-21 18:43:57 +02:00
|
|
|
|
first_wrong_digit_pos = s.pos
|
2020-02-25 22:58:51 +01:00
|
|
|
|
first_wrong_digit = c
|
2020-02-25 14:33:29 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos++
|
|
|
|
|
}
|
2020-04-18 17:49:27 +02:00
|
|
|
|
}
|
|
|
|
|
if has_wrong_digit {
|
2020-07-04 15:14:30 +02:00
|
|
|
|
// error check: wrong digit
|
2020-05-21 18:43:57 +02:00
|
|
|
|
s.pos = first_wrong_digit_pos // adjust error position
|
2020-07-04 15:14:30 +02:00
|
|
|
|
s.error('this number has unsuitable digit `$first_wrong_digit.str()`')
|
|
|
|
|
} else if s.text[s.pos - 1] in [`e`, `E`] {
|
|
|
|
|
// error check: 5e
|
2020-05-21 15:20:36 +02:00
|
|
|
|
s.pos-- // adjust error position
|
2020-04-18 17:49:27 +02:00
|
|
|
|
s.error('exponent has no digits')
|
2020-09-22 05:28:29 +02:00
|
|
|
|
} else if s.pos < s.text.len && s.text[s.pos] == `.` && !is_range && !call_method {
|
2020-07-04 15:14:30 +02:00
|
|
|
|
// error check: 1.23.4, 123.e+3.4
|
2020-04-18 17:49:27 +02:00
|
|
|
|
if has_exp {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.error('exponential part should be integer')
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.error('too many decimal points in number')
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-12-23 13:48:43 +01:00
|
|
|
|
number := s.num_lit(start_pos, s.pos)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos--
|
|
|
|
|
return number
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_number() string {
|
2020-01-23 03:28:25 +01:00
|
|
|
|
if s.expect('0b', s.pos) {
|
|
|
|
|
return s.ident_bin_number()
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if s.expect('0x', s.pos) {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
return s.ident_hex_number()
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else if s.expect('0o', s.pos) {
|
2020-02-23 12:33:07 +01:00
|
|
|
|
return s.ident_oct_number()
|
2020-07-04 15:14:30 +02:00
|
|
|
|
} else {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
return s.ident_dec_number()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-05-03 13:14:32 +02:00
|
|
|
|
[direct_array_access; inline]
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) skip_whitespace() {
|
2021-05-03 13:14:32 +02:00
|
|
|
|
for s.pos < s.text.len {
|
|
|
|
|
c := s.text[s.pos]
|
2021-05-05 22:45:23 +02:00
|
|
|
|
if !(c == 32 || (c > 8 && c < 14) || (c == 0x85) || (c == 0xa0)) {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
return
|
|
|
|
|
}
|
2021-05-03 13:14:32 +02:00
|
|
|
|
c_is_nl := c == scanner.b_cr || c == scanner.b_lf
|
|
|
|
|
if c_is_nl && s.is_vh {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if s.pos + 1 < s.text.len && c == scanner.b_cr && s.text[s.pos + 1] == scanner.b_lf {
|
2021-03-23 06:23:46 +01:00
|
|
|
|
s.is_crlf = true
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
// Count \r\n as one line
|
2021-05-03 13:14:32 +02:00
|
|
|
|
if c_is_nl && !(s.pos > 0 && s.text[s.pos - 1] == scanner.b_cr && c == scanner.b_lf) {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.inc_line_number()
|
|
|
|
|
}
|
|
|
|
|
s.pos++
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) end_of_file() token.Token {
|
2020-05-28 18:22:11 +02:00
|
|
|
|
s.eofs++
|
|
|
|
|
if s.eofs > 50 {
|
|
|
|
|
s.line_nr--
|
2021-01-23 09:33:22 +01:00
|
|
|
|
panic(
|
|
|
|
|
'the end of file `$s.file_path` has been reached 50 times already, the v parser is probably stuck.\n' +
|
2020-07-04 15:14:30 +02:00
|
|
|
|
'This should not happen. Please report the bug here, and include the last 2-3 lines of your source code:\n' +
|
|
|
|
|
'https://github.com/vlang/v/issues/new?labels=Bug&template=bug_report.md')
|
2020-05-28 18:22:11 +02:00
|
|
|
|
}
|
|
|
|
|
if s.pos != s.text.len && s.eofs == 1 {
|
|
|
|
|
s.inc_line_number()
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos = s.text.len
|
2021-02-24 19:03:53 +01:00
|
|
|
|
return s.new_eof_token()
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
|
2021-02-24 19:03:53 +01:00
|
|
|
|
pub fn (mut s Scanner) scan_all_tokens_in_buffer(mode CommentsMode) {
|
2020-06-06 17:47:16 +02:00
|
|
|
|
// s.scan_all_tokens_in_buffer is used mainly by vdoc,
|
|
|
|
|
// in order to implement the .toplevel_comments mode.
|
2021-02-24 19:03:53 +01:00
|
|
|
|
util.timing_start('SCAN')
|
|
|
|
|
defer {
|
|
|
|
|
util.timing_measure_cumulative('SCAN')
|
2020-06-06 17:47:16 +02:00
|
|
|
|
}
|
2021-02-24 19:03:53 +01:00
|
|
|
|
oldmode := s.comments_mode
|
|
|
|
|
s.comments_mode = mode
|
|
|
|
|
s.scan_remaining_text()
|
|
|
|
|
s.comments_mode = oldmode
|
2020-06-06 17:47:16 +02:00
|
|
|
|
s.tidx = 0
|
|
|
|
|
$if debugscanner ? {
|
|
|
|
|
for t in s.all_tokens {
|
2020-07-04 15:14:30 +02:00
|
|
|
|
eprintln('> tidx:${t.tidx:-5} | kind: ${t.kind:-10} | lit: $t.lit')
|
2020-06-06 17:47:16 +02:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-24 19:03:53 +01:00
|
|
|
|
pub fn (mut s Scanner) scan_remaining_text() {
|
|
|
|
|
for {
|
|
|
|
|
t := s.text_scan()
|
|
|
|
|
if s.comments_mode == .skip_comments && t.kind == .comment {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
s.all_tokens << t
|
|
|
|
|
if t.kind == .eof {
|
|
|
|
|
break
|
|
|
|
|
}
|
2020-06-06 17:47:16 +02:00
|
|
|
|
}
|
2021-02-24 19:03:53 +01:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn (mut s Scanner) scan() token.Token {
|
|
|
|
|
return s.buffer_scan()
|
2020-06-06 17:47:16 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn (mut s Scanner) buffer_scan() token.Token {
|
|
|
|
|
for {
|
|
|
|
|
cidx := s.tidx
|
|
|
|
|
s.tidx++
|
|
|
|
|
if cidx >= s.all_tokens.len {
|
|
|
|
|
return s.end_of_file()
|
|
|
|
|
}
|
|
|
|
|
if s.all_tokens[cidx].kind == .comment {
|
|
|
|
|
if !s.should_parse_comment() {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return s.all_tokens[cidx]
|
|
|
|
|
}
|
2021-03-23 08:51:09 +01:00
|
|
|
|
return s.new_eof_token()
|
2020-06-06 17:47:16 +02:00
|
|
|
|
}
|
|
|
|
|
|
2020-06-13 00:01:44 +02:00
|
|
|
|
[inline]
|
2021-02-24 19:03:53 +01:00
|
|
|
|
pub fn (s &Scanner) peek_token(n int) token.Token {
|
|
|
|
|
idx := s.tidx + n
|
|
|
|
|
if idx >= s.all_tokens.len {
|
|
|
|
|
return s.new_eof_token()
|
|
|
|
|
}
|
|
|
|
|
t := s.all_tokens[idx]
|
|
|
|
|
return t
|
|
|
|
|
}
|
|
|
|
|
|
2021-05-05 22:45:23 +02:00
|
|
|
|
[direct_array_access; inline]
|
2021-02-24 19:03:53 +01:00
|
|
|
|
fn (s &Scanner) look_ahead(n int) byte {
|
2020-06-13 00:01:44 +02:00
|
|
|
|
if s.pos + n < s.text.len {
|
|
|
|
|
return s.text[s.pos + n]
|
|
|
|
|
} else {
|
|
|
|
|
return `\0`
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-06-06 17:47:16 +02:00
|
|
|
|
fn (mut s Scanner) text_scan() token.Token {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// The for loop here is so that instead of doing
|
|
|
|
|
// `return s.scan()` (which will use a new call stack frame),
|
|
|
|
|
// text_scan can just do continue, keeping
|
|
|
|
|
// memory & stack usage low.
|
|
|
|
|
// That optimization mostly matters for long sections
|
|
|
|
|
// of comments and string literals.
|
|
|
|
|
for {
|
|
|
|
|
// if s.comments_mode == .parse_comments {
|
|
|
|
|
// println('\nscan()')
|
|
|
|
|
// }
|
|
|
|
|
// if s.line_comment != '' {
|
|
|
|
|
// s.fgenln('// LC "$s.line_comment"')
|
|
|
|
|
// s.line_comment = ''
|
|
|
|
|
// }
|
|
|
|
|
if s.is_started {
|
|
|
|
|
s.pos++
|
2020-12-07 06:33:41 +01:00
|
|
|
|
} else {
|
|
|
|
|
s.is_started = true
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
if !s.is_inside_string {
|
|
|
|
|
s.skip_whitespace()
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2021-01-22 23:24:48 +01:00
|
|
|
|
if s.pos >= s.text.len {
|
|
|
|
|
return s.end_of_file()
|
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// End of $var, start next string
|
|
|
|
|
if s.is_inter_end {
|
|
|
|
|
if s.text[s.pos] == s.quote {
|
|
|
|
|
s.is_inter_end = false
|
|
|
|
|
return s.new_token(.string, '', 1)
|
|
|
|
|
}
|
|
|
|
|
s.is_inter_end = false
|
|
|
|
|
ident_string := s.ident_string()
|
|
|
|
|
return s.new_token(.string, ident_string, ident_string.len + 2) // + two quotes
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.skip_whitespace()
|
|
|
|
|
// end of file
|
|
|
|
|
if s.pos >= s.text.len {
|
|
|
|
|
return s.end_of_file()
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// handle each char
|
|
|
|
|
c := s.text[s.pos]
|
|
|
|
|
nextc := s.look_ahead(1)
|
|
|
|
|
// name or keyword
|
|
|
|
|
if util.is_name_char(c) {
|
|
|
|
|
name := s.ident_name()
|
|
|
|
|
// tmp hack to detect . in ${}
|
|
|
|
|
// Check if not .eof to prevent panic
|
|
|
|
|
next_char := s.look_ahead(1)
|
|
|
|
|
kind := token.keywords[name]
|
|
|
|
|
if kind != .unknown {
|
|
|
|
|
return s.new_token(kind, name, name.len)
|
2020-02-23 23:43:04 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// 'asdf $b' => "b" is the last name in the string, dont start parsing string
|
|
|
|
|
// at the next ', skip it
|
|
|
|
|
if s.is_inside_string {
|
|
|
|
|
if next_char == s.quote {
|
|
|
|
|
s.is_inter_end = true
|
|
|
|
|
s.is_inter_start = false
|
|
|
|
|
s.is_inside_string = false
|
|
|
|
|
}
|
2020-02-23 23:43:04 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// end of `$expr`
|
|
|
|
|
// allow `'$a.b'` and `'$a.c()'`
|
2021-01-23 09:33:22 +01:00
|
|
|
|
if s.is_inter_start && next_char == `\\`
|
2021-02-15 16:54:30 +01:00
|
|
|
|
&& s.look_ahead(2) !in [`x`, `n`, `r`, `\\`, `t`, `e`, `"`, `\'`] {
|
2020-11-27 03:17:07 +01:00
|
|
|
|
s.warn('unknown escape sequence \\${s.look_ahead(2)}')
|
2020-11-27 03:07:37 +01:00
|
|
|
|
}
|
2020-08-22 05:48:06 +02:00
|
|
|
|
if s.is_inter_start && next_char == `(` {
|
|
|
|
|
if s.look_ahead(2) != `)` {
|
2020-11-27 03:07:37 +01:00
|
|
|
|
s.warn('use `\${f(expr)}` instead of `\$f(expr)`')
|
2020-08-22 05:48:06 +02:00
|
|
|
|
}
|
|
|
|
|
} else if s.is_inter_start && next_char != `.` {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.is_inter_end = true
|
|
|
|
|
s.is_inter_start = false
|
2020-07-02 00:02:59 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
if s.pos == 0 && next_char == ` ` {
|
|
|
|
|
// If a single letter name at the start of the file, increment
|
|
|
|
|
// Otherwise the scanner would be stuck at s.pos = 0
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos++
|
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
return s.new_token(.name, name, name.len)
|
|
|
|
|
} else if c.is_digit() || (c == `.` && nextc.is_digit()) {
|
|
|
|
|
// `123`, `.123`
|
|
|
|
|
if !s.is_inside_string {
|
|
|
|
|
// In C ints with `0` prefix are octal (in V they're decimal), so discarding heading zeros is needed.
|
|
|
|
|
mut start_pos := s.pos
|
|
|
|
|
for start_pos < s.text.len && s.text[start_pos] == `0` {
|
|
|
|
|
start_pos++
|
|
|
|
|
}
|
|
|
|
|
mut prefix_zero_num := start_pos - s.pos // how many prefix zeros should be jumped
|
|
|
|
|
// for 0b, 0o, 0x the heading zero shouldn't be jumped
|
|
|
|
|
if start_pos == s.text.len || (c == `0` && !s.text[start_pos].is_digit()) {
|
|
|
|
|
prefix_zero_num--
|
|
|
|
|
}
|
|
|
|
|
s.pos += prefix_zero_num // jump these zeros
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
num := s.ident_number()
|
|
|
|
|
return s.new_token(.number, num, num.len)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// Handle `'$fn()'`
|
|
|
|
|
if c == `)` && s.is_inter_start {
|
|
|
|
|
next_char := s.look_ahead(1)
|
|
|
|
|
if next_char != `.` {
|
|
|
|
|
s.is_inter_end = true
|
|
|
|
|
s.is_inter_start = false
|
|
|
|
|
if next_char == s.quote {
|
|
|
|
|
s.is_inside_string = false
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.rpar, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// all other tokens
|
|
|
|
|
match c {
|
|
|
|
|
`+` {
|
|
|
|
|
if nextc == `+` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.inc, '', 2)
|
|
|
|
|
} else if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.plus_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.plus, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`-` {
|
|
|
|
|
if nextc == `-` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.dec, '', 2)
|
|
|
|
|
} else if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.minus_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.minus, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`*` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.mult_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.mul, '', 1)
|
2020-07-11 11:41:39 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`^` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.xor_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.xor, '', 1)
|
2020-07-11 12:14:10 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`%` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.mod_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.mod, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`?` {
|
|
|
|
|
return s.new_token(.question, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2021-01-25 10:26:20 +01:00
|
|
|
|
scanner.single_quote, scanner.double_quote {
|
2021-04-06 15:16:19 +02:00
|
|
|
|
start_line := s.line_nr
|
2020-04-10 00:09:34 +02:00
|
|
|
|
ident_string := s.ident_string()
|
2021-04-06 15:16:19 +02:00
|
|
|
|
return s.new_multiline_token(.string, ident_string, ident_string.len + 2,
|
|
|
|
|
start_line) // + two quotes
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`\`` {
|
|
|
|
|
// ` // apostrophe balance comment. do not remove
|
|
|
|
|
ident_char := s.ident_char()
|
|
|
|
|
return s.new_token(.chartoken, ident_char, ident_char.len + 2) // + two quotes
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`(` {
|
|
|
|
|
// TODO `$if vet {` for performance
|
|
|
|
|
if s.pref.is_vet && s.text[s.pos + 1] == ` ` {
|
2021-01-09 15:11:49 +01:00
|
|
|
|
s.vet_error('Looks like you are adding a space after `(`', .vfmt)
|
2020-07-11 19:52:05 +02:00
|
|
|
|
}
|
|
|
|
|
return s.new_token(.lpar, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`)` {
|
|
|
|
|
// TODO `$if vet {` for performance
|
|
|
|
|
if s.pref.is_vet && s.text[s.pos - 1] == ` ` {
|
2021-01-09 15:11:49 +01:00
|
|
|
|
s.vet_error('Looks like you are adding a space before `)`', .vfmt)
|
2020-07-11 19:52:05 +02:00
|
|
|
|
}
|
|
|
|
|
return s.new_token(.rpar, '', 1)
|
2020-06-18 18:48:23 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`[` {
|
|
|
|
|
return s.new_token(.lsbr, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`]` {
|
|
|
|
|
return s.new_token(.rsbr, '', 1)
|
2020-05-25 18:33:41 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`{` {
|
|
|
|
|
// Skip { in `${` in strings
|
|
|
|
|
if s.is_inside_string {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.lcbr, '', 1)
|
2020-05-26 15:35:11 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`$` {
|
|
|
|
|
if s.is_inside_string {
|
|
|
|
|
return s.new_token(.str_dollar, '', 1)
|
|
|
|
|
} else {
|
|
|
|
|
return s.new_token(.dollar, '', 1)
|
|
|
|
|
}
|
2020-03-28 21:51:45 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`}` {
|
|
|
|
|
// s = `hello $name !`
|
|
|
|
|
// s = `hello ${name} !`
|
2020-12-09 19:58:27 +01:00
|
|
|
|
if s.is_enclosed_inter {
|
2020-12-10 16:04:12 +01:00
|
|
|
|
if s.pos < s.text.len - 1 {
|
|
|
|
|
s.pos++
|
|
|
|
|
} else {
|
|
|
|
|
s.error('unfinished string literal')
|
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
if s.text[s.pos] == s.quote {
|
|
|
|
|
s.is_inside_string = false
|
2020-12-09 19:58:27 +01:00
|
|
|
|
s.is_enclosed_inter = false
|
2020-07-11 19:52:05 +02:00
|
|
|
|
return s.new_token(.string, '', 1)
|
|
|
|
|
}
|
2020-12-09 19:58:27 +01:00
|
|
|
|
s.is_enclosed_inter = false
|
2020-07-11 19:52:05 +02:00
|
|
|
|
ident_string := s.ident_string()
|
|
|
|
|
return s.new_token(.string, ident_string, ident_string.len + 2) // + two quotes
|
|
|
|
|
} else {
|
|
|
|
|
return s.new_token(.rcbr, '', 1)
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`&` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.and_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
afternextc := s.look_ahead(2)
|
|
|
|
|
if nextc == `&` && afternextc.is_space() {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.and, '', 2)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.amp, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`|` {
|
|
|
|
|
if nextc == `|` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.logical_or, '', 2)
|
|
|
|
|
}
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.or_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.pipe, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`,` {
|
|
|
|
|
return s.new_token(.comma, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`@` {
|
2020-12-11 18:14:07 +01:00
|
|
|
|
mut name := ''
|
|
|
|
|
if nextc != `\0` {
|
|
|
|
|
s.pos++
|
|
|
|
|
name = s.ident_name()
|
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
if s.is_fmt {
|
|
|
|
|
return s.new_token(.name, '@' + name, name.len + 1)
|
|
|
|
|
}
|
2020-11-05 09:12:32 +01:00
|
|
|
|
// @FN, @STRUCT, @MOD etc. See full list in token.valid_at_tokens
|
|
|
|
|
if '@' + name in token.valid_at_tokens {
|
|
|
|
|
return s.new_token(.at, '@' + name, name.len + 1)
|
2020-05-26 22:39:15 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
if !token.is_key(name) {
|
2020-11-05 11:59:49 +01:00
|
|
|
|
mut at_error_msg := '@ must be used before keywords or compile time variables (e.g. `@type string` or `@FN`)'
|
|
|
|
|
// If name is all uppercase, the user is probably looking for a compile time variable ("at-token")
|
|
|
|
|
if name.is_upper() {
|
|
|
|
|
at_error_msg += '\nAvailable compile time variables:\n$token.valid_at_tokens'
|
|
|
|
|
}
|
|
|
|
|
s.error(at_error_msg)
|
2020-07-11 19:52:05 +02:00
|
|
|
|
}
|
|
|
|
|
return s.new_token(.name, name, name.len)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`.` {
|
|
|
|
|
if nextc == `.` {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos++
|
2020-12-10 17:17:25 +01:00
|
|
|
|
if s.pos + 1 < s.text.len && s.text[s.pos + 1] == `.` {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.ellipsis, '', 3)
|
|
|
|
|
}
|
|
|
|
|
return s.new_token(.dotdot, '', 2)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
return s.new_token(.dot, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`#` {
|
|
|
|
|
start := s.pos + 1
|
|
|
|
|
s.ignore_line()
|
|
|
|
|
if nextc == `!` {
|
|
|
|
|
// treat shebang line (#!) as a comment
|
2021-01-05 15:14:16 +01:00
|
|
|
|
comment := s.text[start - 1..s.pos].trim_space()
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// s.fgenln('// shebang line "$s.line_comment"')
|
2021-01-05 15:14:16 +01:00
|
|
|
|
return s.new_token(.comment, comment, comment.len + 2)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
hash := s.text[start..s.pos].trim_space()
|
2021-03-10 19:23:17 +01:00
|
|
|
|
return s.new_token(.hash, hash, hash.len + 2)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`>` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.ge, '', 2)
|
|
|
|
|
} else if nextc == `>` {
|
|
|
|
|
if s.pos + 2 < s.text.len && s.text[s.pos + 2] == `=` {
|
|
|
|
|
s.pos += 2
|
|
|
|
|
return s.new_token(.right_shift_assign, '', 3)
|
|
|
|
|
}
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.right_shift, '', 2)
|
|
|
|
|
} else {
|
|
|
|
|
return s.new_token(.gt, '', 1)
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`<` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.le, '', 2)
|
|
|
|
|
} else if nextc == `<` {
|
|
|
|
|
if s.pos + 2 < s.text.len && s.text[s.pos + 2] == `=` {
|
|
|
|
|
s.pos += 2
|
|
|
|
|
return s.new_token(.left_shift_assign, '', 3)
|
|
|
|
|
}
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.left_shift, '', 2)
|
2020-08-17 20:12:00 +02:00
|
|
|
|
} else if nextc == `-` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.arrow, '', 2)
|
2020-07-11 19:52:05 +02:00
|
|
|
|
} else {
|
|
|
|
|
return s.new_token(.lt, '', 1)
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`=` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.eq, '', 2)
|
|
|
|
|
} else {
|
|
|
|
|
return s.new_token(.assign, '', 1)
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`:` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.decl_assign, '', 2)
|
|
|
|
|
} else {
|
|
|
|
|
return s.new_token(.colon, '', 1)
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`;` {
|
|
|
|
|
return s.new_token(.semicolon, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`!` {
|
|
|
|
|
if nextc == `=` {
|
|
|
|
|
s.pos++
|
|
|
|
|
return s.new_token(.ne, '', 2)
|
2021-01-23 09:33:22 +01:00
|
|
|
|
} else if s.text.len > s.pos + 3 && nextc == `i` && s.text[s.pos + 2] == `n`
|
|
|
|
|
&& s.text[s.pos + 3].is_space() {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.pos += 2
|
|
|
|
|
return s.new_token(.not_in, '', 3)
|
2021-01-23 09:33:22 +01:00
|
|
|
|
} else if s.text.len > s.pos + 3 && nextc == `i` && s.text[s.pos + 2] == `s`
|
|
|
|
|
&& s.text[s.pos + 3].is_space() {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.pos += 2
|
|
|
|
|
return s.new_token(.not_is, '', 3)
|
|
|
|
|
} else {
|
|
|
|
|
return s.new_token(.not, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
`~` {
|
|
|
|
|
return s.new_token(.bit_not, '', 1)
|
|
|
|
|
}
|
|
|
|
|
`/` {
|
|
|
|
|
if nextc == `=` {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos++
|
2020-07-11 19:52:05 +02:00
|
|
|
|
return s.new_token(.div_assign, '', 2)
|
|
|
|
|
}
|
|
|
|
|
if nextc == `/` {
|
|
|
|
|
start := s.pos + 1
|
|
|
|
|
s.ignore_line()
|
2020-08-26 10:39:32 +02:00
|
|
|
|
mut comment_line_end := s.pos
|
2021-05-03 13:14:32 +02:00
|
|
|
|
if s.text[s.pos - 1] == scanner.b_cr {
|
2020-08-26 10:39:32 +02:00
|
|
|
|
comment_line_end--
|
|
|
|
|
} else {
|
|
|
|
|
// fix line_nr, \n was read; the comment is marked on the next line
|
|
|
|
|
s.pos--
|
|
|
|
|
s.line_nr--
|
2020-09-09 15:34:41 +02:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
if s.should_parse_comment() {
|
2020-07-11 20:27:39 +02:00
|
|
|
|
s.line_comment = s.text[start + 1..comment_line_end]
|
2020-12-15 04:26:28 +01:00
|
|
|
|
mut comment := s.line_comment
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// Find out if this comment is on its own line (for vfmt)
|
|
|
|
|
mut is_separate_line_comment := true
|
2021-05-03 13:14:32 +02:00
|
|
|
|
for j := start - 2; j >= 0 && s.text[j] != scanner.b_lf; j-- {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
if s.text[j] !in [`\t`, ` `] {
|
|
|
|
|
is_separate_line_comment = false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if is_separate_line_comment {
|
2021-02-22 17:43:54 +01:00
|
|
|
|
// NB: ´\x01´ is used to preserve the initial whitespace in comments
|
|
|
|
|
// that are on a separate line
|
2020-10-15 22:12:59 +02:00
|
|
|
|
comment = '\x01' + comment
|
2020-07-11 19:52:05 +02:00
|
|
|
|
}
|
|
|
|
|
return s.new_token(.comment, comment, comment.len + 2)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// s.fgenln('// ${s.prev_tok.str()} "$s.line_comment"')
|
|
|
|
|
// Skip the comment (return the next token)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
// Multiline comments
|
|
|
|
|
if nextc == `*` {
|
|
|
|
|
start := s.pos + 2
|
2021-01-19 14:49:40 +01:00
|
|
|
|
start_line := s.line_nr
|
2020-07-11 19:52:05 +02:00
|
|
|
|
mut nest_count := 1
|
|
|
|
|
// Skip comment
|
2020-12-12 04:06:09 +01:00
|
|
|
|
for nest_count > 0 && s.pos < s.text.len - 1 {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.pos++
|
|
|
|
|
if s.pos >= s.text.len {
|
|
|
|
|
s.line_nr--
|
|
|
|
|
s.error('comment not terminated')
|
|
|
|
|
}
|
2021-05-03 13:14:32 +02:00
|
|
|
|
if s.text[s.pos] == scanner.b_lf {
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.inc_line_number()
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if s.expect('/*', s.pos) {
|
|
|
|
|
nest_count++
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if s.expect('*/', s.pos) {
|
|
|
|
|
nest_count--
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
s.pos++
|
|
|
|
|
if s.should_parse_comment() {
|
2021-01-13 03:30:24 +01:00
|
|
|
|
mut comment := s.text[start..(s.pos - 1)].trim(' ')
|
|
|
|
|
if !comment.contains('\n') {
|
|
|
|
|
comment = '\x01' + comment
|
|
|
|
|
}
|
2021-01-27 13:53:41 +01:00
|
|
|
|
return s.new_multiline_token(.comment, comment, comment.len + 4,
|
2021-01-19 14:49:40 +01:00
|
|
|
|
start_line)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
// Skip if not in fmt mode
|
|
|
|
|
continue
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
return s.new_token(.div, '', 1)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
else {}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-07-11 19:52:05 +02:00
|
|
|
|
$if windows {
|
|
|
|
|
if c == `\0` {
|
|
|
|
|
return s.end_of_file()
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2021-02-18 14:43:39 +01:00
|
|
|
|
s.invalid_character()
|
2020-07-11 19:52:05 +02:00
|
|
|
|
break
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
return s.end_of_file()
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-18 14:43:39 +01:00
|
|
|
|
fn (mut s Scanner) invalid_character() {
|
2021-03-01 14:35:57 +01:00
|
|
|
|
len := utf8_char_len(s.text[s.pos])
|
2021-03-12 10:28:04 +01:00
|
|
|
|
end := mu.min(s.pos + len, s.text.len)
|
2021-02-18 14:43:39 +01:00
|
|
|
|
c := s.text[s.pos..end]
|
|
|
|
|
s.error('invalid character `$c`')
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-22 02:34:37 +01:00
|
|
|
|
fn (s &Scanner) current_column() int {
|
|
|
|
|
return s.pos - s.last_nl_pos
|
|
|
|
|
}
|
|
|
|
|
|
2020-04-27 15:08:04 +02:00
|
|
|
|
fn (s &Scanner) count_symbol_before(p int, sym byte) int {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
mut count := 0
|
|
|
|
|
for i := p; i >= 0; i-- {
|
|
|
|
|
if s.text[i] != sym {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
count++
|
|
|
|
|
}
|
|
|
|
|
return count
|
|
|
|
|
}
|
|
|
|
|
|
2021-05-05 22:45:23 +02:00
|
|
|
|
[direct_array_access]
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_string() string {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
q := s.text[s.pos]
|
2021-01-25 10:26:20 +01:00
|
|
|
|
is_quote := q == scanner.single_quote || q == scanner.double_quote
|
2021-01-28 14:38:00 +01:00
|
|
|
|
is_raw := is_quote && s.pos > 0 && s.text[s.pos - 1] == `r` && !s.is_inside_string
|
|
|
|
|
is_cstr := is_quote && s.pos > 0 && s.text[s.pos - 1] == `c` && !s.is_inside_string
|
2020-12-09 19:58:27 +01:00
|
|
|
|
if is_quote {
|
2020-12-11 18:15:59 +01:00
|
|
|
|
if s.is_inside_string || s.is_enclosed_inter || s.is_inter_start {
|
2020-12-09 19:58:27 +01:00
|
|
|
|
s.inter_quote = q
|
|
|
|
|
} else {
|
|
|
|
|
s.quote = q
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
// if s.file_path.contains('string_test') {
|
|
|
|
|
// println('\nident_string() at char=${s.text[s.pos].str()}')
|
|
|
|
|
// println('linenr=$s.line_nr quote= $qquote ${qquote.str()}')
|
|
|
|
|
// }
|
2020-06-23 20:10:51 +02:00
|
|
|
|
mut n_cr_chars := 0
|
2019-12-22 02:34:37 +01:00
|
|
|
|
mut start := s.pos
|
2021-04-12 15:03:22 +02:00
|
|
|
|
start_char := s.text[start]
|
|
|
|
|
if start_char == s.quote
|
|
|
|
|
|| (start_char == s.inter_quote && (s.is_inter_start || s.is_enclosed_inter)) {
|
2020-12-09 19:58:27 +01:00
|
|
|
|
start++
|
2021-05-03 13:14:32 +02:00
|
|
|
|
} else if start_char == scanner.b_lf {
|
2021-04-12 15:03:22 +02:00
|
|
|
|
s.inc_line_number()
|
2020-12-09 19:58:27 +01:00
|
|
|
|
}
|
2020-04-02 12:23:18 +02:00
|
|
|
|
s.is_inside_string = false
|
2021-03-14 11:09:17 +01:00
|
|
|
|
mut u_escapes_pos := []int{} // pos list of \uXXXX
|
2019-12-22 02:34:37 +01:00
|
|
|
|
slash := `\\`
|
|
|
|
|
for {
|
|
|
|
|
s.pos++
|
|
|
|
|
if s.pos >= s.text.len {
|
2020-10-01 15:40:59 +02:00
|
|
|
|
s.error('unfinished string literal')
|
2020-12-09 20:09:07 +01:00
|
|
|
|
break
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
c := s.text[s.pos]
|
|
|
|
|
prevc := s.text[s.pos - 1]
|
|
|
|
|
// end of string
|
|
|
|
|
if c == s.quote && (prevc != slash || (prevc == slash && s.text[s.pos - 2] == slash)) {
|
|
|
|
|
// handle '123\\' slash at the end
|
|
|
|
|
break
|
|
|
|
|
}
|
2020-12-09 19:58:27 +01:00
|
|
|
|
if c == s.inter_quote && (s.is_inter_start || s.is_enclosed_inter) {
|
|
|
|
|
break
|
|
|
|
|
}
|
2021-05-03 13:14:32 +02:00
|
|
|
|
if c == scanner.b_cr {
|
2020-06-23 20:10:51 +02:00
|
|
|
|
n_cr_chars++
|
|
|
|
|
}
|
2021-05-03 13:14:32 +02:00
|
|
|
|
if c == scanner.b_lf {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.inc_line_number()
|
|
|
|
|
}
|
|
|
|
|
// Don't allow \0
|
2020-12-09 02:09:46 +01:00
|
|
|
|
if c == `0` && s.pos > 2 && prevc == slash {
|
2021-01-23 09:33:22 +01:00
|
|
|
|
if (s.pos < s.text.len - 1 && s.text[s.pos + 1].is_digit())
|
|
|
|
|
|| s.count_symbol_before(s.pos - 1, slash) % 2 == 0 {
|
2020-09-19 18:09:18 +02:00
|
|
|
|
} else if !is_cstr && !is_raw {
|
2020-09-21 16:34:24 +02:00
|
|
|
|
s.error(r'cannot use `\0` (NULL character) in the string literal')
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Don't allow \x00
|
|
|
|
|
if c == `0` && s.pos > 5 && s.expect('\\x0', s.pos - 3) {
|
2020-09-18 01:02:06 +02:00
|
|
|
|
if s.count_symbol_before(s.pos - 3, slash) % 2 == 0 {
|
2020-09-19 18:09:18 +02:00
|
|
|
|
} else if !is_cstr && !is_raw {
|
2020-09-21 16:34:24 +02:00
|
|
|
|
s.error(r'cannot use `\x00` (NULL character) in the string literal')
|
2020-07-10 20:41:57 +02:00
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-11-29 16:01:40 +01:00
|
|
|
|
// Escape `\x` `\u`
|
|
|
|
|
if prevc == slash && !is_raw && !is_cstr && s.count_symbol_before(s.pos - 2, slash) % 2 == 0 {
|
|
|
|
|
// Escape `\x`
|
|
|
|
|
if c == `x` && (s.text[s.pos + 1] == s.quote || !s.text[s.pos + 1].is_hex_digit()) {
|
|
|
|
|
s.error(r'`\x` used with no following hex digits')
|
|
|
|
|
}
|
|
|
|
|
// Escape `\u`
|
2021-03-13 07:02:42 +01:00
|
|
|
|
if c == `u` {
|
|
|
|
|
if s.text[s.pos + 1] == s.quote || s.text[s.pos + 2] == s.quote
|
|
|
|
|
|| s.text[s.pos + 3] == s.quote || s.text[s.pos + 4] == s.quote
|
|
|
|
|
|| !s.text[s.pos + 1].is_hex_digit() || !s.text[s.pos + 2].is_hex_digit()
|
|
|
|
|
|| !s.text[s.pos + 3].is_hex_digit() || !s.text[s.pos + 4].is_hex_digit() {
|
|
|
|
|
s.error(r'`\u` incomplete unicode character value')
|
|
|
|
|
}
|
2021-03-14 11:09:17 +01:00
|
|
|
|
u_escapes_pos << s.pos - 1
|
2020-11-29 16:01:40 +01:00
|
|
|
|
}
|
2020-11-28 23:39:45 +01:00
|
|
|
|
}
|
2020-11-27 03:07:37 +01:00
|
|
|
|
// ${var} (ignore in vfmt mode) (skip \$)
|
2020-08-22 05:48:06 +02:00
|
|
|
|
if prevc == `$` && c == `{` && !is_raw && s.count_symbol_before(s.pos - 2, slash) % 2 == 0 {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
s.is_inside_string = true
|
2020-12-09 19:58:27 +01:00
|
|
|
|
s.is_enclosed_inter = true
|
2019-12-22 02:34:37 +01:00
|
|
|
|
// so that s.pos points to $ at the next step
|
|
|
|
|
s.pos -= 2
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
// $var
|
2021-01-23 09:33:22 +01:00
|
|
|
|
if prevc == `$` && util.is_name_char(c) && !is_raw
|
|
|
|
|
&& s.count_symbol_before(s.pos - 2, slash) % 2 == 0 {
|
2020-04-02 12:23:18 +02:00
|
|
|
|
s.is_inside_string = true
|
|
|
|
|
s.is_inter_start = true
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos -= 2
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
mut lit := ''
|
|
|
|
|
mut end := s.pos
|
2020-04-02 12:23:18 +02:00
|
|
|
|
if s.is_inside_string {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
end++
|
|
|
|
|
}
|
2020-05-04 10:19:23 +02:00
|
|
|
|
if start <= s.pos {
|
2020-06-23 20:10:51 +02:00
|
|
|
|
mut string_so_far := s.text[start..end]
|
2021-03-14 11:09:17 +01:00
|
|
|
|
if !s.is_fmt && u_escapes_pos.len > 0 {
|
|
|
|
|
string_so_far = decode_u_escapes(string_so_far, start, u_escapes_pos)
|
2021-03-13 07:02:42 +01:00
|
|
|
|
}
|
2020-06-23 20:10:51 +02:00
|
|
|
|
if n_cr_chars > 0 {
|
|
|
|
|
string_so_far = string_so_far.replace('\r', '')
|
|
|
|
|
}
|
|
|
|
|
if string_so_far.contains('\\\n') {
|
|
|
|
|
lit = trim_slash_line_break(string_so_far)
|
2020-05-04 10:19:23 +02:00
|
|
|
|
} else {
|
2020-06-23 20:10:51 +02:00
|
|
|
|
lit = string_so_far
|
2020-05-04 10:19:23 +02:00
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
return lit
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-14 11:09:17 +01:00
|
|
|
|
fn decode_u_escapes(s string, start int, escapes_pos []int) string {
|
|
|
|
|
if escapes_pos.len == 0 {
|
|
|
|
|
return s
|
|
|
|
|
}
|
|
|
|
|
mut ss := []string{cap: escapes_pos.len * 2 + 1}
|
|
|
|
|
ss << s[..escapes_pos.first() - start]
|
|
|
|
|
for i, pos in escapes_pos {
|
|
|
|
|
idx := pos - start
|
|
|
|
|
end_idx := idx + 6 // "\uXXXX".len == 6
|
|
|
|
|
ss << utf32_to_str(u32(strconv.parse_uint(s[idx + 2..end_idx], 16, 32)))
|
|
|
|
|
if i + 1 < escapes_pos.len {
|
|
|
|
|
ss << s[end_idx..escapes_pos[i + 1] - start]
|
|
|
|
|
} else {
|
|
|
|
|
ss << s[end_idx..]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return ss.join('')
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-04 10:19:23 +02:00
|
|
|
|
fn trim_slash_line_break(s string) string {
|
|
|
|
|
mut start := 0
|
|
|
|
|
mut ret_str := s
|
|
|
|
|
for {
|
|
|
|
|
idx := ret_str.index_after('\\\n', start)
|
|
|
|
|
if idx != -1 {
|
2020-07-04 15:14:30 +02:00
|
|
|
|
ret_str = ret_str[..idx] + ret_str[idx + 2..].trim_left(' \n\t\v\f\r')
|
2020-05-04 10:19:23 +02:00
|
|
|
|
start = idx
|
|
|
|
|
} else {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return ret_str
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ident_char() string {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
start := s.pos
|
|
|
|
|
slash := `\\`
|
|
|
|
|
mut len := 0
|
|
|
|
|
for {
|
|
|
|
|
s.pos++
|
|
|
|
|
if s.pos >= s.text.len {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
if s.text[s.pos] != slash {
|
|
|
|
|
len++
|
|
|
|
|
}
|
|
|
|
|
double_slash := s.expect('\\\\', s.pos - 2)
|
|
|
|
|
if s.text[s.pos] == `\`` && (s.text[s.pos - 1] != slash || double_slash) {
|
|
|
|
|
// ` // apostrophe balance comment. do not remove
|
|
|
|
|
if double_slash {
|
|
|
|
|
len++
|
|
|
|
|
}
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
len--
|
|
|
|
|
c := s.text[start + 1..s.pos]
|
|
|
|
|
if len != 1 {
|
|
|
|
|
u := c.ustring()
|
|
|
|
|
if u.len != 1 {
|
2021-01-23 09:33:22 +01:00
|
|
|
|
s.error('invalid character literal (more than one character)\n' +
|
|
|
|
|
'use quotes for strings, backticks for characters')
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Escapes a `'` character
|
2021-03-22 15:45:29 +01:00
|
|
|
|
if c == "'" {
|
|
|
|
|
return '\\' + c
|
|
|
|
|
}
|
|
|
|
|
return c
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
|
2021-05-05 22:45:23 +02:00
|
|
|
|
[direct_array_access; inline]
|
2019-12-22 02:34:37 +01:00
|
|
|
|
fn (s &Scanner) expect(want string, start_pos int) bool {
|
|
|
|
|
end_pos := start_pos + want.len
|
2020-12-22 10:56:29 +01:00
|
|
|
|
if start_pos < 0 || end_pos < 0 || start_pos >= s.text.len || end_pos > s.text.len {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
for pos in start_pos .. end_pos {
|
|
|
|
|
if s.text[pos] != want[pos - start_pos] {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true
|
|
|
|
|
}
|
|
|
|
|
|
2020-07-11 20:27:39 +02:00
|
|
|
|
[inline]
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) ignore_line() {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.eat_to_end_of_line()
|
|
|
|
|
s.inc_line_number()
|
|
|
|
|
}
|
|
|
|
|
|
2021-05-03 13:14:32 +02:00
|
|
|
|
[direct_array_access; inline]
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) eat_to_end_of_line() {
|
2021-05-03 13:14:32 +02:00
|
|
|
|
for s.pos < s.text.len && s.text[s.pos] != scanner.b_lf {
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.pos++
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-07-11 20:27:39 +02:00
|
|
|
|
[inline]
|
2020-05-17 13:51:18 +02:00
|
|
|
|
fn (mut s Scanner) inc_line_number() {
|
2021-03-23 06:23:46 +01:00
|
|
|
|
s.last_nl_pos = mu.min(s.text.len - 1, s.pos)
|
|
|
|
|
if s.is_crlf {
|
|
|
|
|
s.last_nl_pos++
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
s.line_nr++
|
|
|
|
|
s.line_ends << s.pos
|
|
|
|
|
if s.line_nr > s.nr_lines {
|
|
|
|
|
s.nr_lines = s.line_nr
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-22 18:43:06 +01:00
|
|
|
|
pub fn (mut s Scanner) note(msg string) {
|
|
|
|
|
pos := token.Position{
|
|
|
|
|
line_nr: s.line_nr
|
|
|
|
|
pos: s.pos
|
|
|
|
|
}
|
|
|
|
|
if s.pref.output_mode == .stdout {
|
|
|
|
|
eprintln(util.formatted_error('notice:', msg, s.file_path, pos))
|
|
|
|
|
} else {
|
|
|
|
|
s.notices << errors.Notice{
|
|
|
|
|
file_path: s.file_path
|
|
|
|
|
pos: pos
|
|
|
|
|
reporter: .scanner
|
|
|
|
|
message: msg
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-20 10:44:19 +01:00
|
|
|
|
pub fn (mut s Scanner) warn(msg string) {
|
2020-12-04 14:11:35 +01:00
|
|
|
|
if s.pref.warns_are_errors {
|
|
|
|
|
s.error(msg)
|
|
|
|
|
return
|
|
|
|
|
}
|
2020-08-22 05:48:06 +02:00
|
|
|
|
pos := token.Position{
|
|
|
|
|
line_nr: s.line_nr
|
|
|
|
|
pos: s.pos
|
2021-03-23 06:23:46 +01:00
|
|
|
|
col: s.current_column() - 1
|
2020-08-22 05:48:06 +02:00
|
|
|
|
}
|
2020-11-20 10:44:19 +01:00
|
|
|
|
if s.pref.output_mode == .stdout {
|
|
|
|
|
eprintln(util.formatted_error('warning:', msg, s.file_path, pos))
|
|
|
|
|
} else {
|
|
|
|
|
s.warnings << errors.Warning{
|
|
|
|
|
file_path: s.file_path
|
|
|
|
|
pos: pos
|
|
|
|
|
reporter: .scanner
|
|
|
|
|
message: msg
|
|
|
|
|
}
|
|
|
|
|
}
|
2020-08-22 05:48:06 +02:00
|
|
|
|
}
|
|
|
|
|
|
2020-11-20 10:44:19 +01:00
|
|
|
|
pub fn (mut s Scanner) error(msg string) {
|
2020-04-06 18:58:21 +02:00
|
|
|
|
pos := token.Position{
|
|
|
|
|
line_nr: s.line_nr
|
|
|
|
|
pos: s.pos
|
2021-03-23 06:23:46 +01:00
|
|
|
|
col: s.current_column() - 1
|
2020-04-07 12:29:11 +02:00
|
|
|
|
}
|
2020-11-20 10:44:19 +01:00
|
|
|
|
if s.pref.output_mode == .stdout {
|
|
|
|
|
eprintln(util.formatted_error('error:', msg, s.file_path, pos))
|
|
|
|
|
exit(1)
|
|
|
|
|
} else {
|
2020-12-04 19:34:05 +01:00
|
|
|
|
if s.pref.fatal_errors {
|
|
|
|
|
exit(1)
|
|
|
|
|
}
|
2020-11-20 10:44:19 +01:00
|
|
|
|
s.errors << errors.Error{
|
|
|
|
|
file_path: s.file_path
|
|
|
|
|
pos: pos
|
|
|
|
|
reporter: .scanner
|
|
|
|
|
message: msg
|
|
|
|
|
}
|
|
|
|
|
}
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
|
|
|
|
|
2021-01-09 15:11:49 +01:00
|
|
|
|
fn (mut s Scanner) vet_error(msg string, fix vet.FixKind) {
|
|
|
|
|
ve := vet.Error{
|
|
|
|
|
message: msg
|
|
|
|
|
file_path: s.file_path
|
|
|
|
|
pos: token.Position{
|
|
|
|
|
line_nr: s.line_nr
|
2021-03-23 06:23:46 +01:00
|
|
|
|
col: s.current_column() - 1
|
2021-01-09 15:11:49 +01:00
|
|
|
|
}
|
|
|
|
|
kind: .error
|
|
|
|
|
fix: fix
|
2021-04-09 12:22:14 +02:00
|
|
|
|
typ: .default
|
2021-01-09 15:11:49 +01:00
|
|
|
|
}
|
|
|
|
|
s.vet_errors << ve
|
2020-07-19 19:58:34 +02:00
|
|
|
|
}
|
|
|
|
|
|
2019-12-22 02:34:37 +01:00
|
|
|
|
pub fn verror(s string) {
|
2020-04-03 17:38:41 +02:00
|
|
|
|
util.verror('scanner error', s)
|
2019-12-22 02:34:37 +01:00
|
|
|
|
}
|
2020-05-25 23:00:48 +02:00
|
|
|
|
|
|
|
|
|
pub fn (mut s Scanner) codegen(newtext string) {
|
2021-02-24 19:03:53 +01:00
|
|
|
|
$if debug_codegen ? {
|
|
|
|
|
eprintln('scanner.codegen:\n $newtext')
|
|
|
|
|
}
|
2020-05-25 23:00:48 +02:00
|
|
|
|
// codegen makes sense only during normal compilation
|
|
|
|
|
// feeding code generated V code to vfmt or vdoc will
|
|
|
|
|
// cause them to output/document ephemeral stuff.
|
|
|
|
|
if s.comments_mode == .skip_comments {
|
2021-02-24 19:03:53 +01:00
|
|
|
|
s.all_tokens.delete_last() // remove .eof from end of .all_tokens
|
2020-05-25 23:00:48 +02:00
|
|
|
|
s.text += newtext
|
2021-02-24 19:03:53 +01:00
|
|
|
|
old_tidx := s.tidx
|
|
|
|
|
s.tidx = s.all_tokens.len
|
|
|
|
|
s.scan_remaining_text()
|
|
|
|
|
s.tidx = old_tidx
|
2020-05-25 23:00:48 +02:00
|
|
|
|
}
|
|
|
|
|
}
|
2021-01-28 14:21:19 +01:00
|
|
|
|
|
|
|
|
|
fn (mut s Scanner) trace(fbase string, message string) {
|
|
|
|
|
if s.file_base == fbase {
|
|
|
|
|
println('> s.trace | ${fbase:-10s} | $message')
|
|
|
|
|
}
|
|
|
|
|
}
|