v.scanner: decode \uXXXX in scanner (#9298)

vlib/v/scanner/scanner.v

@@ -5,6 +5,7 @@ module scanner
 
 import math.mathutil as mu
 import os
+import strconv
 import v.token
 import v.pref
 import v.util
@@ -1101,7 +1102,7 @@ fn (mut s Scanner) ident_string() string {
 		start++
 	}
 	s.is_inside_string = false
-	mut u_to_x_pos := []int{} // pos list to replace \u0020 -> \x20
+	mut u_escapes_pos := []int{} // pos list of \uXXXX
 	slash := `\\`
 	for {
 		s.pos++
@@ -1153,11 +1154,8 @@ fn (mut s Scanner) ident_string() string {
 					|| !s.text[s.pos + 1].is_hex_digit() || !s.text[s.pos + 2].is_hex_digit()
 					|| !s.text[s.pos + 3].is_hex_digit() || !s.text[s.pos + 4].is_hex_digit() {
 					s.error(r'`\u` incomplete unicode character value')
-				} else if s.text[s.pos + 1] == `0` && s.text[s.pos + 2] == `0`
-					&& (`0` <= s.text[s.pos + 3] && s.text[s.pos + 3] < `8`) {
-					// ascii
-					u_to_x_pos << s.pos - 1
 				}
+				u_escapes_pos << s.pos - 1
 			}
 		}
 		// ${var} (ignore in vfmt mode) (skip \$)
@@ -1184,14 +1182,8 @@ fn (mut s Scanner) ident_string() string {
 	}
 	if start <= s.pos {
 		mut string_so_far := s.text[start..end]
-		if u_to_x_pos.len > 0 {
+		if !s.is_fmt && u_escapes_pos.len > 0 {
-			mut ss := []string{cap: u_to_x_pos.len + 1}
+			string_so_far = decode_u_escapes(string_so_far, start, u_escapes_pos)
-			ss << string_so_far[..u_to_x_pos[0] - start]
-			for i in 0 .. u_to_x_pos.len - 1 {
-				ss << r'\x' + string_so_far[u_to_x_pos[i] - start + 4..u_to_x_pos[i + 1] - start]
-			}
-			ss << r'\x' + string_so_far[u_to_x_pos.last() + 4 - start..]
-			string_so_far = ss.join('')
 		}
 		if n_cr_chars > 0 {
 			string_so_far = string_so_far.replace('\r', '')
@@ -1205,6 +1197,25 @@ fn (mut s Scanner) ident_string() string {
 	return lit
 }
 
+fn decode_u_escapes(s string, start int, escapes_pos []int) string {
+	if escapes_pos.len == 0 {
+		return s
+	}
+	mut ss := []string{cap: escapes_pos.len * 2 + 1}
+	ss << s[..escapes_pos.first() - start]
+	for i, pos in escapes_pos {
+		idx := pos - start
+		end_idx := idx + 6 // "\uXXXX".len == 6
+		ss << utf32_to_str(u32(strconv.parse_uint(s[idx + 2..end_idx], 16, 32)))
+		if i + 1 < escapes_pos.len {
+			ss << s[end_idx..escapes_pos[i + 1] - start]
+		} else {
+			ss << s[end_idx..]
+		}
+	}
+	return ss.join('')
+}
+
 fn trim_slash_line_break(s string) string {
 	mut start := 0
 	mut ret_str := s

vlib/v/tests/strings_unicode_test.v

@@ -5,6 +5,6 @@ fn test_raw_string() {
 fn test_escape() {
 	assert '\x20' == ' '
 	assert '\u0020' == ' '
-	// assert '\u00c4' == 'Ä'
+	assert '\u00c4' == 'Ä'
 	assert '\r\n'.bytes() == [byte(0x0d), 0x0a]
 }