From da9b6394e8799282292f928171ee4e4a74710ddc Mon Sep 17 00:00:00 2001
From: penguindark <57967770+penguindark@users.noreply.github.com>
Date: Thu, 23 Jan 2020 03:28:25 +0100
Subject: [PATCH] 0bxxxx binary literal support; _ in literals (1_000_000)

---
 vlib/builtin/js/string.v      |  5 ++++
 vlib/builtin/string.v         |  4 +++
 vlib/compiler/scanner.v       | 54 ++++++++++++++++++++++++++++++-----
 vlib/v/scanner/scanner.v      | 54 ++++++++++++++++++++++++++++++-----
 vlib/v/scanner/scanner_test.v |  9 ++++++
 5 files changed, 112 insertions(+), 14 deletions(-)

diff --git a/vlib/builtin/js/string.v b/vlib/builtin/js/string.v
index 8f8a78717e..34ccd242e7 100644
--- a/vlib/builtin/js/string.v
+++ b/vlib/builtin/js/string.v
@@ -220,6 +220,7 @@ fn (s string) at(idx int) byte {
 	}
 	return s.str[idx]
 }
+
 pub fn (c byte) is_digit() bool {
 	return c >= `0` && c <= `9`
 }
@@ -232,6 +233,10 @@ pub fn (c byte) is_oct_digit() bool {
 	return c >= `0` && c <= `7`
 }
 
+pub fn (c byte) is_bin_digit() bool {
+	return c == `0` || c == `1`
+}
+
 pub fn (c byte) is_letter() bool {
 	return (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`)
 }
diff --git a/vlib/builtin/string.v b/vlib/builtin/string.v
index 2ddc27e2ef..8d5b3266f3 100644
--- a/vlib/builtin/string.v
+++ b/vlib/builtin/string.v
@@ -1068,6 +1068,10 @@ pub fn (c byte) is_oct_digit() bool {
 	return c >= `0` && c <= `7`
 }
 
+pub fn (c byte) is_bin_digit() bool {
+	return c == `0` || c == `1`
+}
+
 pub fn (c byte) is_letter() bool {
 	return (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`)
 }
diff --git a/vlib/compiler/scanner.v b/vlib/compiler/scanner.v
index 47de215ef8..c5fe5101e8 100644
--- a/vlib/compiler/scanner.v
+++ b/vlib/compiler/scanner.v
@@ -109,6 +109,43 @@ fn (s mut Scanner) ident_name() string {
 	return name
 }
 
+const(
+	num_sep = `_`	// char used as number separator
+)
+
+fn filter_num_sep(txt byteptr, start int, end int) string {
+	mut b := malloc(end-start + 1) // add a byte for the endstring 0
+	mut i := start
+	mut i1 := 0
+	for i < end {
+		if txt[i] != num_sep {
+			b[i1]=txt[i]
+			i1++
+		}
+		i++
+	}
+	b[i1]=0 // C string compatibility
+	return string{b,i1}
+}
+
+fn (s mut Scanner) ident_bin_number() string {
+	start_pos := s.pos
+	s.pos += 2 // skip '0b'
+	for {
+		if s.pos >= s.text.len {
+			break
+		}
+		c := s.text[s.pos]
+		if !c.is_bin_digit() && c != num_sep {
+			break
+		}
+		s.pos++
+	}
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
+	s.pos--
+	return number
+}
+
 fn (s mut Scanner) ident_hex_number() string {
 	start_pos := s.pos
 	s.pos += 2 // skip '0x'
@@ -117,12 +154,12 @@ fn (s mut Scanner) ident_hex_number() string {
 			break
 		}
 		c := s.text[s.pos]
-		if !c.is_hex_digit() {
+		if !c.is_hex_digit() && c != num_sep {
 			break
 		}
 		s.pos++
 	}
-	number := s.text[start_pos..s.pos]
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
 	s.pos--
 	return number
 }
@@ -135,7 +172,7 @@ fn (s mut Scanner) ident_oct_number() string {
 		}
 		c := s.text[s.pos]
 		if c.is_digit() {
-			if !c.is_oct_digit() {
+			if !c.is_oct_digit() && c != num_sep {
 				s.error('malformed octal constant')
 			}
 		}
@@ -144,7 +181,7 @@ fn (s mut Scanner) ident_oct_number() string {
 		}
 		s.pos++
 	}
-	number := s.text[start_pos..s.pos]
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
 	s.pos--
 	return number
 }
@@ -152,13 +189,13 @@ fn (s mut Scanner) ident_oct_number() string {
 fn (s mut Scanner) ident_dec_number() string {
 	start_pos := s.pos
 	// scan integer part
-	for s.pos < s.text.len && s.text[s.pos].is_digit() {
+	for s.pos < s.text.len && (s.text[s.pos].is_digit() || s.text[s.pos] == num_sep) {
 		s.pos++
 	}
 	// e.g. 1..9
 	// we just return '1' and don't scan '..9'
 	if s.expect('..', s.pos) {
-		number := s.text[start_pos..s.pos]
+		number := filter_num_sep(s.text.str, start_pos, s.pos)
 		s.pos--
 		return number
 	}
@@ -193,12 +230,15 @@ fn (s mut Scanner) ident_dec_number() string {
 			s.error('too many decimal points in number')
 		}
 	}
-	number := s.text[start_pos..s.pos]
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
 	s.pos--
 	return number
 }
 
 fn (s mut Scanner) ident_number() string {
+	if s.expect('0b', s.pos) {
+		return s.ident_bin_number()
+	}
 	if s.expect('0x', s.pos) {
 		return s.ident_hex_number()
 	}
diff --git a/vlib/v/scanner/scanner.v b/vlib/v/scanner/scanner.v
index c42abde0d8..b1d6aa51ec 100644
--- a/vlib/v/scanner/scanner.v
+++ b/vlib/v/scanner/scanner.v
@@ -97,6 +97,43 @@ fn (s mut Scanner) ident_name() string {
 	return name
 }
 
+const(
+	num_sep = `_`	// char used as number separator
+)
+
+fn filter_num_sep(txt byteptr, start int, end int) string {
+	mut b := malloc(end-start + 1) // add a byte for the endstring 0
+	mut i := start
+	mut i1 := 0
+	for i < end {
+		if txt[i] != num_sep {
+			b[i1]=txt[i]
+			i1++
+		}
+		i++
+	}
+	b[i1]=0 // C string compatibility
+	return string{b,i1}
+}
+
+fn (s mut Scanner) ident_bin_number() string {
+	start_pos := s.pos
+	s.pos += 2 // skip '0b'
+	for {
+		if s.pos >= s.text.len {
+			break
+		}
+		c := s.text[s.pos]
+		if !c.is_bin_digit() && c != num_sep {
+			break
+		}
+		s.pos++
+	}
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
+	s.pos--
+	return number
+}
+
 fn (s mut Scanner) ident_hex_number() string {
 	start_pos := s.pos
 	s.pos += 2 // skip '0x'
@@ -105,12 +142,12 @@ fn (s mut Scanner) ident_hex_number() string {
 			break
 		}
 		c := s.text[s.pos]
-		if !c.is_hex_digit() {
+		if !c.is_hex_digit() && c != num_sep {
 			break
 		}
 		s.pos++
 	}
-	number := s.text[start_pos..s.pos]
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
 	s.pos--
 	return number
 }
@@ -123,7 +160,7 @@ fn (s mut Scanner) ident_oct_number() string {
 		}
 		c := s.text[s.pos]
 		if c.is_digit() {
-			if !c.is_oct_digit() {
+			if !c.is_oct_digit() && c != num_sep {
 				s.error('malformed octal constant')
 			}
 		}
@@ -132,7 +169,7 @@ fn (s mut Scanner) ident_oct_number() string {
 		}
 		s.pos++
 	}
-	number := s.text[start_pos..s.pos]
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
 	s.pos--
 	return number
 }
@@ -140,13 +177,13 @@ fn (s mut Scanner) ident_oct_number() string {
 fn (s mut Scanner) ident_dec_number() string {
 	start_pos := s.pos
 	// scan integer part
-	for s.pos < s.text.len && s.text[s.pos].is_digit() {
+	for s.pos < s.text.len && (s.text[s.pos].is_digit() || s.text[s.pos] != num_sep) {
 		s.pos++
 	}
 	// e.g. 1..9
 	// we just return '1' and don't scan '..9'
 	if s.expect('..', s.pos) {
-		number := s.text[start_pos..s.pos]
+		number := filter_num_sep(s.text.str, start_pos, s.pos)
 		s.pos--
 		return number
 	}
@@ -181,12 +218,15 @@ fn (s mut Scanner) ident_dec_number() string {
 			s.error('too many decimal points in number')
 		}
 	}
-	number := s.text[start_pos..s.pos]
+	number := filter_num_sep(s.text.str, start_pos, s.pos)
 	s.pos--
 	return number
 }
 
 fn (s mut Scanner) ident_number() string {
+	if s.expect('0b', s.pos) {
+		return s.ident_bin_number()
+	}
 	if s.expect('0x', s.pos) {
 		return s.ident_hex_number()
 	}
diff --git a/vlib/v/scanner/scanner_test.v b/vlib/v/scanner/scanner_test.v
index 9375b7c428..fb3216c9a0 100644
--- a/vlib/v/scanner/scanner_test.v
+++ b/vlib/v/scanner/scanner_test.v
@@ -26,5 +26,14 @@ fn test_scan() {
 	assert token_kinds[4] == .number
 	assert token_kinds[5] == .rpar
 
+	// test number costants input format
+	mut c := 0xa_0
+	assert c == 0xa0
+	c = 0b10_01
+	assert c == 9
+	c = 1_000_000
+	assert c == 1000000
+	d := f64(23_000_000e-3)
+	assert int(d) == 23000 
 }