toml: add quote details to ast.Quoted (#11997)
parent
a03693e881
commit
bc4aad5fb4
|
@ -140,8 +140,10 @@ pub fn (n Null) str() string {
|
|||
// Quoted types can appear both as keys and values in TOML documents.
|
||||
pub struct Quoted {
|
||||
pub:
|
||||
text string
|
||||
pos token.Position
|
||||
text string
|
||||
pos token.Position
|
||||
is_multiline bool
|
||||
quote byte
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Quoted` type.
|
||||
|
@ -149,6 +151,8 @@ pub fn (q Quoted) str() string {
|
|||
mut str := typeof(q).name + '{\n'
|
||||
str += ' text: \'$q.text\'\n'
|
||||
str += ' pos: $q.pos\n'
|
||||
str += ' is_multiline: $q.is_multiline\n'
|
||||
str += ' quote: \'$q.quote\'\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
|
|
@ -691,9 +691,19 @@ pub fn (mut p Parser) bare() ast.Bare {
|
|||
|
||||
// quoted parse and returns an `ast.Quoted` type.
|
||||
pub fn (mut p Parser) quoted() ast.Quoted {
|
||||
// To get more info about the quote type and enable better checking,
|
||||
// the scanner is returning the literal *with* single- or double-quotes.
|
||||
mut quote := p.tok.lit[0]
|
||||
is_multiline := p.tok.lit.len >= 6 && p.tok.lit[1] == quote && p.tok.lit[2] == quote
|
||||
mut lit := p.tok.lit[1..p.tok.lit.len - 1]
|
||||
if is_multiline {
|
||||
lit = p.tok.lit[3..p.tok.lit.len - 3]
|
||||
}
|
||||
return ast.Quoted{
|
||||
text: p.tok.lit
|
||||
text: lit
|
||||
pos: p.tok.position()
|
||||
quote: quote
|
||||
is_multiline: is_multiline
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -145,10 +145,9 @@ pub fn (mut s Scanner) scan() ?token.Token {
|
|||
return s.new_token(.assign, ascii, ascii.len)
|
||||
}
|
||||
`"`, `'` { // ... some string "/'
|
||||
ident_string, is_multiline := s.extract_string() ?
|
||||
token_length := if is_multiline { 2 * 3 } else { 2 }
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'identified quoted string (multiline: $is_multiline) `$ident_string`')
|
||||
return s.new_token(.quoted, ident_string, ident_string.len + token_length) // + quote length
|
||||
ident_string := s.extract_string() ?
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'identified quoted string `$ident_string`')
|
||||
return s.new_token(.quoted, ident_string, ident_string.len)
|
||||
}
|
||||
`#` {
|
||||
start := s.pos //+ 1
|
||||
|
@ -333,20 +332,20 @@ fn (mut s Scanner) extract_key() string {
|
|||
// any bytes recognized as a TOML string.
|
||||
// TOML strings are everything found between two double or single quotation marks (`"`/`'`).
|
||||
[direct_array_access; inline]
|
||||
fn (mut s Scanner) extract_string() ?(string, bool) {
|
||||
fn (mut s Scanner) extract_string() ?string {
|
||||
// extract_string is called when the scanner has already reached
|
||||
// a byte that is the start of a string so we rewind it to start at the correct
|
||||
s.pos--
|
||||
s.col--
|
||||
quote := s.at()
|
||||
start := s.pos
|
||||
mut lit := ''
|
||||
mut lit := quote.ascii_str()
|
||||
|
||||
is_multiline := s.text[s.pos + 1] == quote && s.text[s.pos + 2] == quote
|
||||
// Check for escaped multiline quote
|
||||
if is_multiline {
|
||||
mls := s.extract_multiline_string() ?
|
||||
return mls, is_multiline
|
||||
return mls
|
||||
}
|
||||
|
||||
for {
|
||||
|
@ -375,7 +374,7 @@ fn (mut s Scanner) extract_string() ?(string, bool) {
|
|||
if c == quote {
|
||||
s.pos++
|
||||
s.col++
|
||||
return lit, is_multiline
|
||||
return lit + quote.ascii_str()
|
||||
}
|
||||
|
||||
lit += c.ascii_str()
|
||||
|
@ -386,8 +385,7 @@ fn (mut s Scanner) extract_string() ?(string, bool) {
|
|||
' unfinished single-line string literal `$quote.ascii_str()` started at $start ($s.line_nr,$s.col) "${byte(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
}
|
||||
|
||||
return lit, is_multiline
|
||||
return lit
|
||||
}
|
||||
|
||||
// extract_multiline_string collects and returns a string containing
|
||||
|
@ -399,7 +397,7 @@ fn (mut s Scanner) extract_multiline_string() ?string {
|
|||
// characters is the quotes
|
||||
quote := s.at()
|
||||
start := s.pos
|
||||
mut lit := ''
|
||||
mut lit := quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
|
||||
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'multi-line `$quote.ascii_str()${s.text[s.pos +
|
||||
1].ascii_str()}${s.text[s.pos + 2].ascii_str()}` string started at pos $start ($s.line_nr,$s.col) (quote type: $quote.ascii_str() / $quote)')
|
||||
|
@ -441,6 +439,7 @@ fn (mut s Scanner) extract_multiline_string() ?string {
|
|||
if s.peek(3) == -1 {
|
||||
s.pos += 3
|
||||
s.col += 3
|
||||
lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'returning at $c.ascii_str() `$lit`')
|
||||
return lit
|
||||
} else if s.peek(3) != quote {
|
||||
|
@ -448,6 +447,7 @@ fn (mut s Scanner) extract_multiline_string() ?string {
|
|||
// lit += quote.ascii_str()
|
||||
s.pos += 3
|
||||
s.col += 3
|
||||
lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'returning at $c.ascii_str() `$lit`')
|
||||
return lit
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue