vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

parser,cgen: handle `const x = $embed_file("file.txt").to_string()`

pull/13272/head
Delyan Angelov 2022-01-25 10:58:23 +02:00
parent 32f2f0dfa1
commit 5f38ba896e
No known key found for this signature in database
GPG Key ID: 66886C0F12D595ED
10 changed files with 102 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CONTRIBUTING.md
View File

@ -199,3 +199,4 @@ to create a copy of the compiler rather than replacing it with `v self`.
| `trace_parser` | Prints details about parsed statements and expressions |
| `trace_thirdparty_obj_files` | Prints details about built thirdparty obj files |
| `trace_usecache` | Prints details when -usecache is used |
| `trace_embed_file` | Prints details when $embed_file is used |

10
vlib/v/ast/embed_file.v 100644
View File

@ -0,0 +1,10 @@
// Copyright (c) 2019-2022 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module ast
import hash.fnv1a
pub fn (e EmbeddedFile) hash() u64 {
return fnv1a.sum64_string('$e.apath, $e.compression_type, $e.is_compressed, $e.len')
}

8
vlib/v/embed_file/tests/embed_file_test.v
View File

@ -1,5 +1,13 @@
const const_file = $embed_file('v.png')
const src = $embed_file('embed_file_test.v').to_string()
fn test_const_embed_file_to_string() {
assert src.len > 0
assert src.split_into_lines()[0].starts_with('const const_file')
assert src.split_into_lines().last() == '}'
}
fn test_const_embed_file() {
mut file := const_file
eprintln('file: $file')

7
vlib/v/gen/c/cgen.v
View File

@ -762,8 +762,11 @@ pub fn (mut g Gen) finish() {
if g.pref.is_livemain || g.pref.is_liveshared {
g.generate_hotcode_reloader_code()
}
if g.embed_file_is_prod_mode() && g.embedded_files.len > 0 {
g.gen_embedded_data()
if g.embedded_files.len > 0 {
if g.embed_file_is_prod_mode() {
g.gen_embedded_data()
}
g.gen_embedded_metadata()
}
if g.pref.is_test {
g.gen_c_main_for_tests()

98
vlib/v/gen/c/embed.v
View File

@ -14,6 +14,9 @@ fn (mut g Gen) embed_file_is_prod_mode() bool {
// gen_embed_file_struct generates C code for `$embed_file('...')` calls.
fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
$if trace_embed_file ? {
eprintln('> gen_embed_file_init $node.embed_file.apath')
}
if g.embed_file_is_prod_mode() {
file_bytes := os.read_bytes(node.embed_file.apath) or {
panic('unable to read file: "$node.embed_file.rpath')
@ -31,7 +34,11 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
cache_path := os.join_path(cache_dir, cache_key)
vexe := pref.vexe_path()
result := os.execute('${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}')
compress_cmd := '${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}'
$if trace_embed_file ? {
eprintln('> gen_embed_file_init, compress_cmd: $compress_cmd')
}
result := os.execute(compress_cmd)
if result.exit_code != 0 {
eprintln('unable to compress file "$node.embed_file.rpath": $result.output')
node.embed_file.bytes = file_bytes
@ -57,41 +64,62 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
}
node.embed_file.len = file_bytes.len
}
g.writeln('(v__embed_file__EmbedFileData){')
g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},')
if g.embed_file_is_prod_mode() {
// apath is not needed in production and may leak information
g.writeln('\t\t.apath = ${ctoslit('')},')
} else {
g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},')
}
if g.embed_file_is_prod_mode() {
// use function generated in Gen.gen_embedded_data()
if node.embed_file.is_compressed {
g.writeln('\t\t.compression_type = ${ctoslit(node.embed_file.compression_type)},')
g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,')
g.writeln('\t\t.uncompressed = NULL,')
} else {
g.writeln('\t\t.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,')
}
} else {
g.writeln('\t\t.uncompressed = NULL,')
}
g.writeln('\t\t.free_compressed = 0,')
g.writeln('\t\t.free_uncompressed = 0,')
if g.embed_file_is_prod_mode() {
g.writeln('\t\t.len = $node.embed_file.len')
} else {
file_size := os.file_size(node.embed_file.apath)
if file_size > 5242880 {
eprintln('Warning: embedding of files >= ~5MB is currently not supported')
}
g.writeln('\t\t.len = $file_size')
}
g.writeln('} // \$embed_file("$node.embed_file.apath")')
ef_idx := node.embed_file.hash()
g.write('_v_embed_file_metadata($ef_idx)')
g.file.embedded_files << node.embed_file
$if trace_embed_file ? {
eprintln('> gen_embed_file_init => _v_embed_file_metadata(${ef_idx:-25}) | ${node.embed_file.apath:-50} | compression: $node.embed_file.compression_type | len: $node.embed_file.len')
}
}
// gen_embedded_metadata embeds all of the deduplicated metadata in g.embedded_files, into the V target executable,
// into a single generated function _v_embed_file_metadata, that accepts a hash of the absolute path of the embedded
// files.
fn (mut g Gen) gen_embedded_metadata() {
g.embedded_data.writeln('v__embed_file__EmbedFileData _v_embed_file_metadata(u64 ef_hash) {')
g.embedded_data.writeln('\tv__embed_file__EmbedFileData res;')
g.embedded_data.writeln('\tmemset(&res, 0, sizeof(res));')
g.embedded_data.writeln('\tswitch(ef_hash) {')
for emfile in g.embedded_files {
ef_idx := emfile.hash()
g.embedded_data.writeln('\t\tcase $ef_idx: {')
g.embedded_data.writeln('\t\t\tres.path = ${ctoslit(emfile.rpath)};')
if g.embed_file_is_prod_mode() {
// apath is not needed in production and may leak information
g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit('')};')
} else {
g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit(emfile.apath)};')
}
if g.embed_file_is_prod_mode() {
// use function generated in Gen.gen_embedded_data()
if emfile.is_compressed {
g.embedded_data.writeln('\t\t\tres.compression_type = ${ctoslit(emfile.compression_type)};')
g.embedded_data.writeln('\t\t\tres.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;')
g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;')
} else {
g.embedded_data.writeln('\t\t\tres.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;')
}
} else {
g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;')
}
g.embedded_data.writeln('\t\t\tres.free_compressed = 0;')
g.embedded_data.writeln('\t\t\tres.free_uncompressed = 0;')
if g.embed_file_is_prod_mode() {
g.embedded_data.writeln('\t\t\tres.len = $emfile.len;')
} else {
file_size := os.file_size(emfile.apath)
if file_size > 5242880 {
eprintln('Warning: embedding of files >= ~5MB is currently not supported')
}
g.embedded_data.writeln('\t\t\tres.len = $file_size;')
}
g.embedded_data.writeln('\t\t\tbreak;')
g.embedded_data.writeln('\t\t} // case $ef_idx')
}
g.embedded_data.writeln('\t\tdefault: _v_panic(_SLIT("unknown embed file"));')
g.embedded_data.writeln('\t} // switch')
g.embedded_data.writeln('\treturn res;')
g.embedded_data.writeln('}')
}
// gen_embedded_data embeds data into the V target executable.

4
vlib/v/gen/c/testdata/embed.c.must_have vendored
View File

@ -13,7 +13,7 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){
.path = _SLIT("embed.vv"),
v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
res.path = _SLIT("embed.vv");
// Initializations for module v.embed_file :

8
vlib/v/gen/c/testdata/embed_with_prod.c.must_have vendored
View File

@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){
.path = _SLIT("embed.vv"),
.apath = _SLIT(""),
.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data,
v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
res.path = _SLIT("embed.vv");
res.apath = _SLIT("");
res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data;
// Initializations for module v.embed_file :

6
vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have vendored
View File

@ -6,6 +6,6 @@ VV_LOCAL_SYMBOL void v__preludes__embed_file__zlib__init(void);
VV_LOCAL_SYMBOL Option_Array_byte v__preludes__embed_file__zlib__ZLibDecoder_decompress(v__preludes__embed_file__zlib__ZLibDecoder _d1, Array_byte data) {
= compress__zlib__decompress(data);
.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data,
.compression_type = _SLIT("zlib")
.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data,
res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data;
res.compression_type = _SLIT("zlib");
res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data;

8
vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have vendored
View File

@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){
.path = _SLIT("embed.vv"),
.apath = _SLIT(""),
.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data,
v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
res.path = _SLIT("embed.vv");
res.apath = _SLIT("");
res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data;
// Initializations for module v.embed_file :

3
vlib/v/parser/expr.v
View File

@ -78,7 +78,8 @@ pub fn (mut p Parser) check_expr(precedence int) ?ast.Expr {
.dollar {
match p.peek_tok.kind {
.name {
return p.comptime_call()
node = p.comptime_call()
p.is_stmt_ident = is_stmt_ident
}
.key_if {
return p.if_expr(true)