vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

toml: small refactor, move all json functionality to submodule (#12502)

pull/12507/head
Larpon 2021-11-18 12:27:59 +01:00 committed by GitHub
parent 5bf28c5287
commit 7ec70d5477
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 223 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/toml.v
View File

@ -1,4 +1,5 @@
import toml
import toml.to
// Complete text from the example in the README.md:
// https://github.com/toml-lang/toml/blob/3b11f6921da7b6f5db37af039aa021fee450c091/README.md#Example
@ -43,6 +44,6 @@ fn main() {
ip := doc.value('servers.alpha.ip').string()
println('Server IP: "$ip"')
toml_json := doc.to_json()
toml_json := to.json(doc)
println(toml_json)
}

3
vlib/toml/README.md
View File

@ -5,6 +5,7 @@
```v
import toml
import toml.to
// Complete text from the example in the README.md:
// https://github.com/toml-lang/toml/blob/3b11f6921da7b6f5db37af039aa021fee450c091/README.md#Example
@ -49,7 +50,7 @@ fn main() {
ip := doc.value('servers.alpha.ip').string()
println('Server IP: "$ip"')
toml_json := doc.to_json()
toml_json := to.json(doc)
println(toml_json)
}
```

108
vlib/toml/any.v
View File

@ -3,9 +3,6 @@
// that can be found in the LICENSE file.
module toml
import toml.util
import x.json2
// Pretty much all the same builtin types as the `json2.Any` type plus `DateTime`,`Date`,`Time`
pub type Any = Date
| DateTime
@ -156,7 +153,7 @@ pub fn (a Any) datetime() DateTime {
// `key` should be in "dotted" form (`a.b.c`).
// `key` supports quoted keys like `a."b.c"`.
pub fn (m map[string]Any) value(key string) ?Any {
key_split := util.parse_dotted_key(key) ?
key_split := parse_dotted_key(key) ?
return m.value_(key_split)
}
@ -182,106 +179,3 @@ pub fn (a []Any) as_strings() []string {
}
return sa
}
// to_json returns `Any` as a JSON encoded string.
pub fn (a Any) to_json() string {
match a {
Null {
return 'null'
}
DateTime {
json_text := json2.Any(a.str())
return '"$json_text.json_str()"'
}
Date {
json_text := json2.Any(a.str())
return '"$json_text.json_str()"'
}
Time {
json_text := json2.Any(a.str())
return '"$json_text.json_str()"'
}
string {
json_text := json2.Any(a.str())
return '"$json_text.json_str()"'
}
bool, f32, f64, i64, int, u64 {
json_text := json2.Any(a.str())
return json_text.json_str()
}
map[string]Any {
mut str := '{'
for key, val in a {
json_key := json2.Any(key)
str += ' "$json_key.json_str()": $val.to_json(),'
}
str = str.trim_right(',')
str += ' }'
return str
}
[]Any {
mut str := '['
for val in a {
str += ' $val.to_json(),'
}
str = str.trim_right(',')
str += ' ]'
return str
}
}
}
// to_json_any returns `Any` as a `x.json2.Any` type.
pub fn (a Any) to_json_any() json2.Any {
match a {
Null {
return json2.Null{}
}
DateTime {
return json2.Any(a.str())
}
Date {
return json2.Any(a.str())
}
Time {
return json2.Any(a.str())
}
string {
return json2.Any(a.str())
}
bool {
return json2.Any(bool(a))
}
int {
return json2.Any(int(a))
}
f32 {
return json2.Any(f32(a))
}
f64 {
return json2.Any(f64(a))
}
i64 {
return json2.Any(i64(a))
}
u64 {
return json2.Any(u64(a))
}
map[string]Any {
mut jmap := map[string]json2.Any{}
for key, val in a {
jmap[key] = val.to_json_any()
}
return jmap
}
[]Any {
mut jarr := []json2.Any{}
for val in a {
jarr << val.to_json_any()
}
return jarr
}
}
}

4
vlib/toml/ast/ast.v
View File

@ -23,7 +23,3 @@ pub fn (r Root) str() string {
s += '}'
return s
}
pub fn (r Root) to_json() string {
return r.table.to_json()
}

16
vlib/toml/ast/types.v
View File

@ -4,7 +4,6 @@
module ast
import toml.token
import x.json2
// Key is a sumtype representing all types of keys that
// can be found in a TOML document.
@ -26,21 +25,20 @@ pub type Value = Bool
| []Value
| map[string]Value
pub fn (v Value) to_json() string {
// str outputs the value in JSON-like format for eased
// debugging
pub fn (v Value) str() string {
match v {
Quoted, Date, DateTime, Time {
json_text := json2.Any(v.text)
return '"$json_text.json_str()"'
return '"$v.text"'
}
Bool, Null, Number {
json_text := json2.Any(v.text)
return json_text.json_str()
return v.text
}
map[string]Value {
mut str := '{'
for key, val in v {
json_key := json2.Any(key)
str += ' "$json_key.json_str()": $val.to_json(),'
str += ' "$key": $val,'
}
str = str.trim_right(',')
str += ' }'
@ -49,7 +47,7 @@ pub fn (v Value) to_json() string {
[]Value {
mut str := '['
for val in v {
str += ' $val.to_json(),'
str += ' $val,'
}
str = str.trim_right(',')
str += ' ]'

22
vlib/toml/parser/parser.v
View File

@ -412,7 +412,7 @@ pub fn (mut p Parser) root_table() ? {
t := p.find_sub_table(sub_table) ?
unsafe {
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'setting "$key" = $val.to_json() in table ${ptr_str(t)}')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'setting "$key" = $val in table ${ptr_str(t)}')
t[key.str()] = val
}
} else {
@ -429,7 +429,7 @@ pub fn (mut p Parser) root_table() ? {
t := p.find_table() ?
unsafe {
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'setting "$key.str()" = $val.to_json() in table ${ptr_str(t)}')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'setting "$key.str()" = $val in table ${ptr_str(t)}')
key_str := key.str()
if _ := t[key_str] {
return error(@MOD + '.' + @STRUCT + '.' + @FN +
@ -577,7 +577,7 @@ pub fn (mut p Parser) inline_table(mut tbl map[string]ast.Value) ? {
mut t := p.find_in_table(mut tbl, sub_table) ?
unsafe {
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @6 "$key" = $val.to_json() into ${ptr_str(t)}')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @6 "$key" = $val into ${ptr_str(t)}')
t[key.str()] = val
}
} else {
@ -588,7 +588,7 @@ pub fn (mut p Parser) inline_table(mut tbl map[string]ast.Value) ? {
return error(@MOD + '.' + @STRUCT + '.' + @FN +
' key "$key_str" is already initialized with a value. At "$p.tok.kind" "$p.tok.lit" in this (excerpt): "...${p.excerpt()}..."')
}
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @5 "$key_str" = $val.to_json() into ${ptr_str(tbl)}')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @5 "$key_str" = $val into ${ptr_str(tbl)}')
tbl[key_str] = val
}
previous_token_was_value = true
@ -674,7 +674,7 @@ pub fn (mut p Parser) array_of_tables_contents() ?[]ast.Value {
mut t := p.find_in_table(mut tbl, sub_table) ?
unsafe {
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @6 "$key" = $val.to_json() into ${ptr_str(t)}')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @6 "$key" = $val into ${ptr_str(t)}')
t[key.str()] = val
}
} else {
@ -689,7 +689,7 @@ pub fn (mut p Parser) array_of_tables_contents() ?[]ast.Value {
}
mut arr := []ast.Value{}
arr << tbl
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed array of tables ${ast.Value(arr).to_json()}. leaving at "$p.tok.kind" "$p.tok.lit"')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed array of tables ${ast.Value(arr)}. leaving at "$p.tok.kind" "$p.tok.lit"')
return arr
}
@ -816,7 +816,7 @@ pub fn (mut p Parser) double_array_of_tables_contents(target_key DottedKey) ?[]a
mut t := p.find_in_table(mut tbl, sub_table) ?
unsafe {
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @6 "$key" = $val.to_json() into ${ptr_str(t)}')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @6 "$key" = $val into ${ptr_str(t)}')
t[key.str()] = val
}
} else {
@ -830,7 +830,7 @@ pub fn (mut p Parser) double_array_of_tables_contents(target_key DottedKey) ?[]a
t = p.find_in_table(mut tbl, implicit_allocation_key) ?
}
unsafe {
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @7 "$key" = $val.to_json() into ${ptr_str(t)}')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'inserting @7 "$key" = $val into ${ptr_str(t)}')
t[key.str()] = val
}
}
@ -870,7 +870,7 @@ pub fn (mut p Parser) double_array_of_tables_contents(target_key DottedKey) ?[]a
}
mut arr := []ast.Value{}
arr << tbl
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed array of tables ${ast.Value(arr).to_json()}. leaving at "$p.tok.kind" "$p.tok.lit"')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed array of tables ${ast.Value(arr)}. leaving at "$p.tok.kind" "$p.tok.lit"')
return arr
}
@ -1041,7 +1041,7 @@ pub fn (mut p Parser) key_value() ?(ast.Key, ast.Value) {
p.check(.assign) ? // Assignment operator
p.ignore_while(parser.space_formatting)
value := p.value() ?
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed key value pair. "$key" = $value.to_json()')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed key value pair. "$key" = $value')
return key, value
}
@ -1085,7 +1085,7 @@ pub fn (mut p Parser) value() ?ast.Value {
' value expected .boolean, .quoted, .lsbr, .lcbr or .number got "$p.tok.kind" "$p.tok.lit" in this (excerpt): "...${p.excerpt()}..."')
}
}
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed "$p.tok.kind" as value $value.to_json()')
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'parsed "$p.tok.kind" as value $value')
return value
}

3
vlib/toml/tests/array_of_tables_1_level_test.v
View File

@ -1,5 +1,6 @@
import os
import toml
import toml.to
const (
toml_table_text = '
@ -19,7 +20,7 @@ color = "gray"'
fn test_tables() {
mut toml_doc := toml.parse(toml_table_text) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
assert toml_json == os.read_file(
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +

3
vlib/toml/tests/array_of_tables_2_level_test.v
View File

@ -1,5 +1,6 @@
import os
import toml
import toml.to
const (
toml_text = '[[albums]]
@ -24,7 +25,7 @@ name = "Born in the USA"
fn test_nested_array_of_tables() {
mut toml_doc := toml.parse(toml_text) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
eprintln(toml_json)
assert toml_json == os.read_file(

3
vlib/toml/tests/array_of_tables_array_test.v
View File

@ -1,5 +1,6 @@
import os
import toml
import toml.to
const (
toml_text = '[[a]]
@ -15,7 +16,7 @@ const (
fn test_nested_array_of_tables() {
mut toml_doc := toml.parse(toml_text) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
eprintln(toml_json)
assert toml_json == os.read_file(

6
vlib/toml/tests/datetime_test.v
View File

@ -56,17 +56,17 @@ fn test_dates() {
od_time := toml.Date{'1979-05-27'}
ld1 := toml_doc.value('ld1')
assert ld1.date() == od_time
// assert ld1.string() == '1979-05-27' // TODO fail in CI but pass locally?
// assert ld1.string() == '1979-05-27' // TODO memory corruption
// lt1 test section
mut ot_time := toml.Time{'07:32:00'}
lt1 := toml_doc.value('lt1')
assert lt1.time() == ot_time
// assert lt1.string() == '07:32:00' // TODO fail in CI but pass locally?
// assert lt1.string() == '07:32:00' // TODO memory corruption
// lt2 test section
ot_time = toml.Time{'00:32:00.999999'}
lt2 := toml_doc.value('lt2')
assert lt2.time() == ot_time
// assert lt2.string() == '00:32:00.999999' // TODO fail in CI but pass locally?
// assert lt2.string() == '00:32:00.999999' // TODO memory corruption
}

5
vlib/toml/tests/json_encoding_test.v
View File

@ -1,5 +1,6 @@
import os
import toml
import toml.to
fn test_parse() {
toml_file :=
@ -7,13 +8,11 @@ fn test_parse() {
'.toml'
toml_doc := toml.parse(toml_file) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
out_file :=
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
'.out'
out_file_json := os.read_file(out_file) or { panic(err) }
println(toml_json)
assert toml_json == out_file_json
// assert false
}

5
vlib/toml/tests/json_test.v
View File

@ -1,5 +1,6 @@
import os
import toml
import toml.to
fn test_parse() {
toml_file :=
@ -7,13 +8,11 @@ fn test_parse() {
'.toml'
toml_doc := toml.parse(toml_file) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
out_file :=
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
'.out'
out_file_json := os.read_file(out_file) or { panic(err) }
println(toml_json)
assert toml_json == out_file_json
// assert false
}

3
vlib/toml/tests/toml_bom_test.v
View File

@ -1,5 +1,6 @@
import os
import toml
import toml.to
import toml.ast
const empty_toml_document = toml.Doc{
@ -17,7 +18,7 @@ const (
fn test_toml_with_bom() {
toml_doc := toml.parse(toml_text_with_utf8_bom) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
title := toml_doc.value('title')
assert title == toml.Any('TOML Example')

9
vlib/toml/tests/toml_test.v
View File

@ -1,5 +1,6 @@
import os
import toml
import toml.to
const toml_text = os.read_file(
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
@ -9,7 +10,7 @@ fn test_toml() {
// File containing the complete text from the example in the official TOML project README.md:
// https://github.com/toml-lang/toml/blob/3b11f6921da7b6f5db37af039aa021fee450c091/README.md#Example
toml_doc := toml.parse(toml_text) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
// NOTE Kept for easier debugging:
// dump(toml_doc.ast)
@ -76,7 +77,7 @@ fn test_toml_file() {
os.write_file(test_file, toml_text) or { assert false }
toml_doc := toml.parse_file(test_file) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
// NOTE Kept for easier debugging:
// dump(toml_doc.ast)
@ -90,7 +91,7 @@ fn test_toml_file() {
fn test_toml_parse_text() {
toml_doc := toml.parse_text(toml_text) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
assert toml_json == os.read_file(
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
'.out') or { panic(err) }
@ -98,7 +99,7 @@ fn test_toml_parse_text() {
fn test_toml_parse() {
toml_doc := toml.parse(toml_text) or { panic(err) }
toml_json := toml_doc.to_json()
toml_json := to.json(toml_doc)
assert toml_json == os.read_file(
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
'.out') or { panic(err) }

137
vlib/toml/to/to.v 100644
View File

@ -0,0 +1,137 @@
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module to
import toml
import x.json2
type DocOrAny = toml.Any | toml.Doc
// json returns `doa` as a JSON encoded string.
pub fn json(doa DocOrAny) string {
match doa {
toml.Doc {
return any_to_json(doa.ast_to_any(doa.ast.table))
}
toml.Any {
return any_to_json(doa)
}
}
}
// json returns `a` as a JSON encoded string.
fn any_to_json(a toml.Any) string {
match a {
toml.Null {
return 'null'
}
toml.DateTime {
json_text := json2.Any(a.str())
return '"$json_text.json_str()"'
}
toml.Date {
json_text := json2.Any(a.str())
return '"$json_text.json_str()"'
}
toml.Time {
json_text := json2.Any(a.str())
return '"$json_text.json_str()"'
}
string {
return '"' + json2.Any(a.str()).json_str() + '"'
}
bool {
return json2.Any(bool(a)).json_str()
}
f32 {
return json2.Any(f32(a)).json_str()
}
f64 {
return json2.Any(f64(a)).json_str()
}
i64 {
return json2.Any(i64(a)).json_str()
}
int {
return json2.Any(int(a)).json_str()
}
u64 {
return json2.Any(u64(a)).json_str()
}
map[string]toml.Any {
mut str := '{'
for key, val in a {
json_key := json2.Any(key)
str += ' "$json_key.json_str()": ${any_to_json(val)},'
}
str = str.trim_right(',')
str += ' }'
return str
}
[]toml.Any {
mut str := '['
for val in a {
str += ' ${any_to_json(val)},'
}
str = str.trim_right(',')
str += ' ]'
return str
}
}
}
// json_any returns `Any` as a `x.json2.Any` type.
pub fn json_any(a toml.Any) json2.Any {
match a {
toml.Null {
return json2.Null{}
}
toml.DateTime {
return json2.Any(a.str())
}
toml.Date {
return json2.Any(a.str())
}
toml.Time {
return json2.Any(a.str())
}
string {
return json2.Any(a.str())
}
bool {
return json2.Any(bool(a))
}
int {
return json2.Any(int(a))
}
f32 {
return json2.Any(f32(a))
}
f64 {
return json2.Any(f64(a))
}
i64 {
return json2.Any(i64(a))
}
u64 {
return json2.Any(u64(a))
}
map[string]toml.Any {
mut jmap := map[string]json2.Any{}
for key, val in a {
jmap[key] = json_any(val)
}
return jmap
}
[]toml.Any {
mut jarr := []json2.Any{}
for val in a {
jarr << json_any(val)
}
return jarr
}
}
}

52
vlib/toml/toml.v
View File

@ -4,7 +4,6 @@
module toml
import toml.ast
import toml.util
import toml.input
import toml.scanner
import toml.parser
@ -14,7 +13,7 @@ import strconv
pub struct Null {
}
// DateTime is the representation of an RFC 3339 date-only string.
// DateTime is the representation of an RFC 3339 datetime string.
pub struct DateTime {
datetime string
}
@ -23,7 +22,7 @@ pub fn (dt DateTime) str() string {
return dt.datetime
}
// Date is the representation of an RFC 3339 datetime string.
// Date is the representation of an RFC 3339 date-only string.
pub struct Date {
date string
}
@ -111,9 +110,45 @@ pub fn parse(toml string) ?Doc {
}
}
// to_json returns a compact json string of the complete document.
pub fn (d Doc) to_json() string {
return d.ast.to_json()
// parse_dotted_key converts `key` string to an array of strings.
// parse_dotted_key preserves strings delimited by both `"` and `'`.
pub fn parse_dotted_key(key string) ?[]string {
mut out := []string{}
mut buf := ''
mut in_string := false
mut delim := byte(` `)
for ch in key {
if ch in [`"`, `'`] {
if !in_string {
delim = ch
}
in_string = !in_string && ch == delim
if !in_string {
if buf != '' && buf != ' ' {
out << buf
}
buf = ''
delim = ` `
}
continue
}
buf += ch.ascii_str()
if !in_string && ch == `.` {
if buf != '' && buf != ' ' {
out << buf[..buf.len - 1]
}
buf = ''
continue
}
}
if buf != '' && buf != ' ' {
out << buf
}
if in_string {
return error(@FN +
': could not parse key, missing closing string delimiter `$delim.ascii_str()`')
}
return out
}
// to_any converts the `Doc` to toml.Any type.
@ -126,13 +161,12 @@ pub fn (d Doc) to_any() Any {
// `key` supports quoted keys like `a."b.c"`.
pub fn (d Doc) value(key string) Any {
values := d.ast.table as map[string]ast.Value
key_split := util.parse_dotted_key(key) or { return Any(Null{}) }
key_split := parse_dotted_key(key) or { return Any(Null{}) }
return d.value_(values, key_split)
}
// value_ returns the value found at `key` in the map `values` as `Any` type.
fn (d Doc) value_(values map[string]ast.Value, key []string) Any {
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, ' getting "${key[0]}"')
value := values[key[0]] or {
return Any(Null{})
// TODO decide this
@ -150,7 +184,7 @@ fn (d Doc) value_(values map[string]ast.Value, key []string) Any {
}
// ast_to_any converts `from` ast.Value to toml.Any value.
fn (d Doc) ast_to_any(value ast.Value) Any {
pub fn (d Doc) ast_to_any(value ast.Value) Any {
match value {
ast.Date {
return Any(Date{value.text})

41
vlib/toml/util/util.v
View File

@ -25,44 +25,3 @@ pub fn is_illegal_ascii_control_character(byte_char byte) bool {
pub fn printdbg(id string, message string) {
eprintln(id + ' ' + message)
}
// parse_dotted_key converts `key` string to an array of strings.
// parse_dotted_key preserves strings delimited by both `"` and `'`.
pub fn parse_dotted_key(key string) ?[]string {
mut out := []string{}
mut buf := ''
mut in_string := false
mut delim := byte(` `)
for ch in key {
if ch in [`"`, `'`] {
if !in_string {
delim = ch
}
in_string = !in_string && ch == delim
if !in_string {
if buf != '' && buf != ' ' {
out << buf
}
buf = ''
delim = ` `
}
continue
}
buf += ch.ascii_str()
if !in_string && ch == `.` {
if buf != '' && buf != ' ' {
out << buf[..buf.len - 1]
}
buf = ''
continue
}
}
if buf != '' && buf != ' ' {
out << buf
}
if in_string {
return error(@FN +
': could not parse key, missing closing string delimiter `$delim.ascii_str()`')
}
return out
}