0bxxxx binary literal support; _ in literals (1_000_000)

pull/3540/head
penguindark 2020-01-23 03:28:25 +01:00 committed by Alexander Medvednikov
parent c3d0814517
commit da9b6394e8
5 changed files with 112 additions and 14 deletions

View File

@ -220,6 +220,7 @@ fn (s string) at(idx int) byte {
} }
return s.str[idx] return s.str[idx]
} }
pub fn (c byte) is_digit() bool { pub fn (c byte) is_digit() bool {
return c >= `0` && c <= `9` return c >= `0` && c <= `9`
} }
@ -232,6 +233,10 @@ pub fn (c byte) is_oct_digit() bool {
return c >= `0` && c <= `7` return c >= `0` && c <= `7`
} }
pub fn (c byte) is_bin_digit() bool {
return c == `0` || c == `1`
}
pub fn (c byte) is_letter() bool { pub fn (c byte) is_letter() bool {
return (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`) return (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`)
} }

View File

@ -1068,6 +1068,10 @@ pub fn (c byte) is_oct_digit() bool {
return c >= `0` && c <= `7` return c >= `0` && c <= `7`
} }
pub fn (c byte) is_bin_digit() bool {
return c == `0` || c == `1`
}
pub fn (c byte) is_letter() bool { pub fn (c byte) is_letter() bool {
return (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`) return (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`)
} }

View File

@ -109,6 +109,43 @@ fn (s mut Scanner) ident_name() string {
return name return name
} }
const(
num_sep = `_` // char used as number separator
)
fn filter_num_sep(txt byteptr, start int, end int) string {
mut b := malloc(end-start + 1) // add a byte for the endstring 0
mut i := start
mut i1 := 0
for i < end {
if txt[i] != num_sep {
b[i1]=txt[i]
i1++
}
i++
}
b[i1]=0 // C string compatibility
return string{b,i1}
}
fn (s mut Scanner) ident_bin_number() string {
start_pos := s.pos
s.pos += 2 // skip '0b'
for {
if s.pos >= s.text.len {
break
}
c := s.text[s.pos]
if !c.is_bin_digit() && c != num_sep {
break
}
s.pos++
}
number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos--
return number
}
fn (s mut Scanner) ident_hex_number() string { fn (s mut Scanner) ident_hex_number() string {
start_pos := s.pos start_pos := s.pos
s.pos += 2 // skip '0x' s.pos += 2 // skip '0x'
@ -117,12 +154,12 @@ fn (s mut Scanner) ident_hex_number() string {
break break
} }
c := s.text[s.pos] c := s.text[s.pos]
if !c.is_hex_digit() { if !c.is_hex_digit() && c != num_sep {
break break
} }
s.pos++ s.pos++
} }
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
@ -135,7 +172,7 @@ fn (s mut Scanner) ident_oct_number() string {
} }
c := s.text[s.pos] c := s.text[s.pos]
if c.is_digit() { if c.is_digit() {
if !c.is_oct_digit() { if !c.is_oct_digit() && c != num_sep {
s.error('malformed octal constant') s.error('malformed octal constant')
} }
} }
@ -144,7 +181,7 @@ fn (s mut Scanner) ident_oct_number() string {
} }
s.pos++ s.pos++
} }
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
@ -152,13 +189,13 @@ fn (s mut Scanner) ident_oct_number() string {
fn (s mut Scanner) ident_dec_number() string { fn (s mut Scanner) ident_dec_number() string {
start_pos := s.pos start_pos := s.pos
// scan integer part // scan integer part
for s.pos < s.text.len && s.text[s.pos].is_digit() { for s.pos < s.text.len && (s.text[s.pos].is_digit() || s.text[s.pos] == num_sep) {
s.pos++ s.pos++
} }
// e.g. 1..9 // e.g. 1..9
// we just return '1' and don't scan '..9' // we just return '1' and don't scan '..9'
if s.expect('..', s.pos) { if s.expect('..', s.pos) {
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
@ -193,12 +230,15 @@ fn (s mut Scanner) ident_dec_number() string {
s.error('too many decimal points in number') s.error('too many decimal points in number')
} }
} }
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
fn (s mut Scanner) ident_number() string { fn (s mut Scanner) ident_number() string {
if s.expect('0b', s.pos) {
return s.ident_bin_number()
}
if s.expect('0x', s.pos) { if s.expect('0x', s.pos) {
return s.ident_hex_number() return s.ident_hex_number()
} }

View File

@ -97,6 +97,43 @@ fn (s mut Scanner) ident_name() string {
return name return name
} }
const(
num_sep = `_` // char used as number separator
)
fn filter_num_sep(txt byteptr, start int, end int) string {
mut b := malloc(end-start + 1) // add a byte for the endstring 0
mut i := start
mut i1 := 0
for i < end {
if txt[i] != num_sep {
b[i1]=txt[i]
i1++
}
i++
}
b[i1]=0 // C string compatibility
return string{b,i1}
}
fn (s mut Scanner) ident_bin_number() string {
start_pos := s.pos
s.pos += 2 // skip '0b'
for {
if s.pos >= s.text.len {
break
}
c := s.text[s.pos]
if !c.is_bin_digit() && c != num_sep {
break
}
s.pos++
}
number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos--
return number
}
fn (s mut Scanner) ident_hex_number() string { fn (s mut Scanner) ident_hex_number() string {
start_pos := s.pos start_pos := s.pos
s.pos += 2 // skip '0x' s.pos += 2 // skip '0x'
@ -105,12 +142,12 @@ fn (s mut Scanner) ident_hex_number() string {
break break
} }
c := s.text[s.pos] c := s.text[s.pos]
if !c.is_hex_digit() { if !c.is_hex_digit() && c != num_sep {
break break
} }
s.pos++ s.pos++
} }
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
@ -123,7 +160,7 @@ fn (s mut Scanner) ident_oct_number() string {
} }
c := s.text[s.pos] c := s.text[s.pos]
if c.is_digit() { if c.is_digit() {
if !c.is_oct_digit() { if !c.is_oct_digit() && c != num_sep {
s.error('malformed octal constant') s.error('malformed octal constant')
} }
} }
@ -132,7 +169,7 @@ fn (s mut Scanner) ident_oct_number() string {
} }
s.pos++ s.pos++
} }
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
@ -140,13 +177,13 @@ fn (s mut Scanner) ident_oct_number() string {
fn (s mut Scanner) ident_dec_number() string { fn (s mut Scanner) ident_dec_number() string {
start_pos := s.pos start_pos := s.pos
// scan integer part // scan integer part
for s.pos < s.text.len && s.text[s.pos].is_digit() { for s.pos < s.text.len && (s.text[s.pos].is_digit() || s.text[s.pos] != num_sep) {
s.pos++ s.pos++
} }
// e.g. 1..9 // e.g. 1..9
// we just return '1' and don't scan '..9' // we just return '1' and don't scan '..9'
if s.expect('..', s.pos) { if s.expect('..', s.pos) {
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
@ -181,12 +218,15 @@ fn (s mut Scanner) ident_dec_number() string {
s.error('too many decimal points in number') s.error('too many decimal points in number')
} }
} }
number := s.text[start_pos..s.pos] number := filter_num_sep(s.text.str, start_pos, s.pos)
s.pos-- s.pos--
return number return number
} }
fn (s mut Scanner) ident_number() string { fn (s mut Scanner) ident_number() string {
if s.expect('0b', s.pos) {
return s.ident_bin_number()
}
if s.expect('0x', s.pos) { if s.expect('0x', s.pos) {
return s.ident_hex_number() return s.ident_hex_number()
} }

View File

@ -26,5 +26,14 @@ fn test_scan() {
assert token_kinds[4] == .number assert token_kinds[4] == .number
assert token_kinds[5] == .rpar assert token_kinds[5] == .rpar
// test number costants input format
mut c := 0xa_0
assert c == 0xa0
c = 0b10_01
assert c == 9
c = 1_000_000
assert c == 1000000
d := f64(23_000_000e-3)
assert int(d) == 23000
} }