v/vlib/builtin/utf8.v

154 lines
3.6 KiB
V
Raw Normal View History

// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved.
2019-06-23 04:21:30 +02:00
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
2019-06-22 20:20:28 +02:00
module builtin
2019-09-15 14:36:05 +02:00
pub fn utf8_char_len(b byte) int {
2019-12-19 21:52:45 +01:00
return ((0xe5000000>>((b>>3) & 0x1e)) & 3) + 1
2019-06-22 20:20:28 +02:00
}
// Convert utf32 to utf8
// utf32 == Codepoint
pub fn utf32_to_str(code u32) string {
unsafe {
mut buffer := malloc(5)
return utf32_to_str_no_malloc(code, buffer)
}
2019-06-22 20:20:28 +02:00
}
[unsafe]
pub fn utf32_to_str_no_malloc(code u32, buf voidptr) string {
2019-12-19 21:52:45 +01:00
icode := int(code) // Prevents doing casts everywhere
2020-11-21 22:33:31 +01:00
mut res := ''
unsafe {
mut buffer := byteptr(buf)
if icode <= 127 { /* 0x7F */
buffer[0] = byte(icode)
2020-11-21 22:33:31 +01:00
res = tos(buffer, 1)
}
2020-11-21 22:37:16 +01:00
else if icode <= 2047 { /* 0x7FF */
buffer[0] = 192 | byte(icode>>6) /* 0xC0 - 110xxxxx */
buffer[1] = 128 | byte(icode & 63) /* 0x80 - 0x3F - 10xxxxxx */
2020-11-21 22:33:31 +01:00
res = tos(buffer, 2)
}
2020-11-21 22:37:16 +01:00
else if icode <= 65535 { /* 0xFFFF */
buffer[0] = 224 | byte(icode>>12)/* 0xE0 - 1110xxxx */
buffer[1] = 128 | (byte(icode>>6) & 63) /* 0x80 - 0x3F - 10xxxxxx */
buffer[2] = 128 | byte(icode & 63) /* 0x80 - 0x3F - 10xxxxxx */
2020-11-21 22:33:31 +01:00
res = tos(buffer, 3)
}
2020-11-21 22:37:16 +01:00
else if icode <= 1114111/* 0x10FFFF */ {
buffer[0] = 240 | byte(icode>>18) /* 0xF0 - 11110xxx */
buffer[1] = 128 | (byte(icode>>12) & 63) /* 0x80 - 0x3F - 10xxxxxx */
buffer[2] = 128 | (byte(icode>>6) & 63) /* 0x80 - 0x3F - 10xxxxxx */
buffer[3] = 128 | byte(icode & 63) /* 0x80 - 0x3F - 10xxxxxx */
2020-11-21 22:33:31 +01:00
res = tos(buffer, 4)
}
}
2020-11-21 22:33:31 +01:00
res.is_lit = 1 // let autofree know this string doesn't have to be freed
return res
2019-06-22 20:20:28 +02:00
}
// Convert utf8 to utf32
pub fn (_rune string) utf32_code() int {
2019-06-22 20:20:28 +02:00
if _rune.len == 0 {
return 0
}
// save ASC symbol as is
if _rune.len == 1 {
return int(_rune[0])
}
mut b := byte(int(_rune[0]))
2019-06-22 20:20:28 +02:00
// TODO should be
// res := int( rune[0] << rune.len)
2019-12-19 21:52:45 +01:00
b = b<<_rune.len
mut res := int(b)
2019-06-22 20:20:28 +02:00
mut shift := 6 - _rune.len
for i := 1; i < _rune.len; i++ {
c := int(_rune[i])
2019-12-19 21:52:45 +01:00
res = res<<shift
res |= c & 63 // 0x3f
2019-06-22 20:20:28 +02:00
shift = 6
}
return res
}
// Calculate length to read from the first byte
fn utf8_len(c byte) int {
2019-12-19 21:52:45 +01:00
mut b := 0
mut x := c
if (x & 240) != 0 {
2019-12-19 21:52:45 +01:00
// 0xF0
x >>= 4
}
else {
b += 4
}
if (x & 12) != 0 {
2019-12-19 21:52:45 +01:00
// 0x0C
x >>= 2
}
else {
b += 2
}
if (x & 2) == 0 {
2019-12-19 21:52:45 +01:00
// 0x02
b++
}
return b
}
// Calculate string length for in number of codepoints
fn utf8_str_len(s string) int {
mut l := 0
for i := 0; i < s.len; i++ {
l++
c := unsafe {s.str[i]}
if (c & (1 << 7)) != 0 {
for t := byte(1 << 6); (c & t) != 0; t >>= 1 {
i++
}
}
}
return l
}
// Calculate string length for formatting, i.e. number of "characters"
fn utf8_str_visible_length(s string) int {
mut l := 0
mut ul := 1
for i := 0; i < s.len; i+=ul {
ul = 1
c := unsafe {s.str[i]}
if (c & (1 << 7)) != 0 {
for t := byte(1 << 6); (c & t) != 0; t >>= 1 {
ul++
}
}
if i + ul > s.len { // incomplete UTF-8 sequence
return l
}
l++
// recognize combining characters
if c == 0xcc || c == 0xcd {
r := (u16(c) << 8) | unsafe {s.str[i+1]}
if r >= 0xcc80 && r < 0xcdb0 { // diacritical marks
l--
}
} else if c == 0xe1 || c == 0xe2 || c == 0xef {
r := (u32(c) << 16) | unsafe {(u32(s.str[i+1]) << 8) | s.str[i+2]}
// diacritical marks extended 0xe1aab0 - 0xe1ac80
// diacritical marks supplement 0xe1b780 - 0xe1b880
// diacritical marks for symbols 0xe28390 - 0xe28480
// half marks 0xefb8a0 - 0xefb8b0
2020-11-21 22:33:31 +01:00
if (r >= 0xe1aab0 && r < 0xe1ac80)
|| (r >= 0xe1b780 && r < 0xe1b880)
|| (r >= 0xe28390 && r < 0xe28480)
|| (r >= 0xefb8a0 && r < 0xefb8b0) {
l--
}
}
}
return l
}