checker: ensure hex character literals don't overflow in strings (#10725)

vlib/v/checker/check_types.v

@@ -493,6 +493,44 @@ pub fn (mut c Checker) string_inter_lit(mut node ast.StringInterLiteral) ast.Typ
 	return ast.string_type
 }
 
+pub fn (mut c Checker) string_lit(mut node ast.StringLiteral) ast.Type {
+	mut idx := 0
+	for idx < node.val.len {
+		match node.val[idx] {
+			`\\` {
+				mut start_pos := token.Position{
+					...node.pos
+					col: node.pos.col + 1 + idx
+				}
+				start_idx := idx
+				idx++
+				next_ch := node.val[idx] or { return ast.string_type }
+				if next_ch == `x` {
+					idx++
+					mut ch := node.val[idx] or { return ast.string_type }
+					mut hex_char_count := 0
+					for ch.is_hex_digit() {
+						hex_char_count++
+						if hex_char_count > 4 {
+							end_pos := token.Position{
+								...start_pos
+								len: idx + 1 - start_idx
+							}
+							c.error('hex character literal overflows string', end_pos)
+						}
+						idx++
+						ch = node.val[idx] or { return ast.string_type }
+					}
+				}
+			}
+			else {
+				idx++
+			}
+		}
+	}
+	return ast.string_type
+}
+
 pub fn (mut c Checker) int_lit(mut node ast.IntegerLiteral) ast.Type {
 	if node.val.len < 17 {
 		// can not be a too large number, no need for more expensive checks

vlib/v/checker/checker.v

@@ -5069,7 +5069,7 @@ pub fn (mut c Checker) expr(node ast.Expr) ast.Type {
 				// string literal starts with "c": `C.printf(c'hello')`
 				return ast.byte_type.set_nr_muls(1)
 			}
-			return ast.string_type
+			return c.string_lit(mut node)
 		}
 		ast.StringInterLiteral {
 			return c.string_inter_lit(mut node)

vlib/v/checker/tests/hex_literal_overflow.out

@@ -0,0 +1,4 @@
+vlib/v/checker/tests/hex_literal_overflow.vv:1:35: error: hex character literal overflows string
+    1 | s := 'this hex character literal, \xffffffff, should be caught by the checker'
+      |                                   ~~~~~~~
+    2 | println(s)

vlib/v/checker/tests/hex_literal_overflow.vv

@@ -0,0 +1,2 @@
+s := 'this hex character literal, \xffffffff, should be caught by the checker'
+println(s)