toml: add quote details to ast.Quoted (#11997)

vlib/toml/ast/types.v

@@ -140,8 +140,10 @@ pub fn (n Null) str() string {
 // Quoted types can appear both as keys and values in TOML documents.
 pub struct Quoted {
 pub:
-	text string
-	pos  token.Position
+	text         string
+	pos          token.Position
+	is_multiline bool
+	quote        byte
 }
 
 // str returns the `string` representation of the `Quoted` type.
@@ -149,6 +151,8 @@ pub fn (q Quoted) str() string {
 	mut str := typeof(q).name + '{\n'
 	str += '  text:  \'$q.text\'\n'
 	str += '  pos:  $q.pos\n'
+	str += '  is_multiline:  $q.is_multiline\n'
+	str += '  quote: \'$q.quote\'\n'
 	str += '}'
 	return str
 }

vlib/toml/parser/parser.v

@@ -691,9 +691,19 @@ pub fn (mut p Parser) bare() ast.Bare {
 
 // quoted parse and returns an `ast.Quoted` type.
 pub fn (mut p Parser) quoted() ast.Quoted {
+	// To get more info about the quote type and enable better checking,
+	// the scanner is returning the literal *with* single- or double-quotes.
+	mut quote := p.tok.lit[0]
+	is_multiline := p.tok.lit.len >= 6 && p.tok.lit[1] == quote && p.tok.lit[2] == quote
+	mut lit := p.tok.lit[1..p.tok.lit.len - 1]
+	if is_multiline {
+		lit = p.tok.lit[3..p.tok.lit.len - 3]
+	}
 	return ast.Quoted{
-		text: p.tok.lit
+		text: lit
 		pos: p.tok.position()
+		quote: quote
+		is_multiline: is_multiline
 	}
 }
 

vlib/toml/scanner/scanner.v

@@ -145,10 +145,9 @@ pub fn (mut s Scanner) scan() ?token.Token {
 				return s.new_token(.assign, ascii, ascii.len)
 			}
 			`"`, `'` { // ... some string "/'
-				ident_string, is_multiline := s.extract_string() ?
-				token_length := if is_multiline { 2 * 3 } else { 2 }
-				util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'identified quoted string (multiline: $is_multiline) `$ident_string`')
-				return s.new_token(.quoted, ident_string, ident_string.len + token_length) // + quote length
+				ident_string := s.extract_string() ?
+				util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'identified quoted string `$ident_string`')
+				return s.new_token(.quoted, ident_string, ident_string.len)
 			}
 			`#` {
 				start := s.pos //+ 1
@@ -333,20 +332,20 @@ fn (mut s Scanner) extract_key() string {
 // any bytes recognized as a TOML string.
 // TOML strings are everything found between two double or single quotation marks (`"`/`'`).
 [direct_array_access; inline]
-fn (mut s Scanner) extract_string() ?(string, bool) {
+fn (mut s Scanner) extract_string() ?string {
 	// extract_string is called when the scanner has already reached
 	// a byte that is the start of a string so we rewind it to start at the correct
 	s.pos--
 	s.col--
 	quote := s.at()
 	start := s.pos
-	mut lit := ''
+	mut lit := quote.ascii_str()
 
 	is_multiline := s.text[s.pos + 1] == quote && s.text[s.pos + 2] == quote
 	// Check for escaped multiline quote
 	if is_multiline {
 		mls := s.extract_multiline_string() ?
-		return mls, is_multiline
+		return mls
 	}
 
 	for {
@@ -375,7 +374,7 @@ fn (mut s Scanner) extract_string() ?(string, bool) {
 		if c == quote {
 			s.pos++
 			s.col++
-			return lit, is_multiline
+			return lit + quote.ascii_str()
 		}
 
 		lit += c.ascii_str()
@@ -386,8 +385,7 @@ fn (mut s Scanner) extract_string() ?(string, bool) {
 				' unfinished single-line string literal `$quote.ascii_str()` started at $start ($s.line_nr,$s.col) "${byte(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
 		}
 	}
-
-	return lit, is_multiline
+	return lit
 }
 
 // extract_multiline_string collects and returns a string containing
@@ -399,7 +397,7 @@ fn (mut s Scanner) extract_multiline_string() ?string {
 	// characters is the quotes
 	quote := s.at()
 	start := s.pos
-	mut lit := ''
+	mut lit := quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
 
 	util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'multi-line `$quote.ascii_str()${s.text[s.pos +
 		1].ascii_str()}${s.text[s.pos + 2].ascii_str()}` string started at pos $start ($s.line_nr,$s.col) (quote type: $quote.ascii_str() / $quote)')
@@ -441,6 +439,7 @@ fn (mut s Scanner) extract_multiline_string() ?string {
 				if s.peek(3) == -1 {
 					s.pos += 3
 					s.col += 3
+					lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
 					util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'returning at $c.ascii_str() `$lit`')
 					return lit
 				} else if s.peek(3) != quote {
@@ -448,6 +447,7 @@ fn (mut s Scanner) extract_multiline_string() ?string {
 					// lit += quote.ascii_str()
 					s.pos += 3
 					s.col += 3
+					lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
 					util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'returning at $c.ascii_str() `$lit`')
 					return lit
 				}