parser,cgen: handle `const x = $embed_file("file.txt").to_string()`

CONTRIBUTING.md

@@ -199,3 +199,4 @@ to create a copy of the compiler rather than replacing it with `v self`.
 | `trace_parser` | Prints details about parsed statements and expressions |
 | `trace_thirdparty_obj_files` | Prints details about built thirdparty obj files |
 | `trace_usecache` | Prints details when -usecache is used |
+| `trace_embed_file` | Prints details when $embed_file is used |

vlib/v/ast/embed_file.v

@@ -0,0 +1,10 @@
+// Copyright (c) 2019-2022 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module ast
+
+import hash.fnv1a
+
+pub fn (e EmbeddedFile) hash() u64 {
+	return fnv1a.sum64_string('$e.apath, $e.compression_type, $e.is_compressed, $e.len')
+}

vlib/v/embed_file/tests/embed_file_test.v

@@ -1,5 +1,13 @@
 const const_file = $embed_file('v.png')
 
+const src = $embed_file('embed_file_test.v').to_string()
+
+fn test_const_embed_file_to_string() {
+	assert src.len > 0
+	assert src.split_into_lines()[0].starts_with('const const_file')
+	assert src.split_into_lines().last() == '}'
+}
+
 fn test_const_embed_file() {
 	mut file := const_file
 	eprintln('file: $file')

vlib/v/gen/c/cgen.v

@@ -762,9 +762,12 @@ pub fn (mut g Gen) finish() {
 	if g.pref.is_livemain || g.pref.is_liveshared {
 		g.generate_hotcode_reloader_code()
 	}
-	if g.embed_file_is_prod_mode() && g.embedded_files.len > 0 {
+	if g.embedded_files.len > 0 {
+		if g.embed_file_is_prod_mode() {
 			g.gen_embedded_data()
 		}
+		g.gen_embedded_metadata()
+	}
 	if g.pref.is_test {
 		g.gen_c_main_for_tests()
 	} else {

vlib/v/gen/c/embed.v

@@ -14,6 +14,9 @@ fn (mut g Gen) embed_file_is_prod_mode() bool {
 
 // gen_embed_file_struct generates C code for `$embed_file('...')` calls.
 fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
+	$if trace_embed_file ? {
+		eprintln('> gen_embed_file_init $node.embed_file.apath')
+	}
 	if g.embed_file_is_prod_mode() {
 		file_bytes := os.read_bytes(node.embed_file.apath) or {
 			panic('unable to read file: "$node.embed_file.rpath')
@@ -31,7 +34,11 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
 			cache_path := os.join_path(cache_dir, cache_key)
 
 			vexe := pref.vexe_path()
-			result := os.execute('${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}')
+			compress_cmd := '${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}'
+			$if trace_embed_file ? {
+				eprintln('> gen_embed_file_init, compress_cmd: $compress_cmd')
+			}
+			result := os.execute(compress_cmd)
 			if result.exit_code != 0 {
 				eprintln('unable to compress file "$node.embed_file.rpath": $result.output')
 				node.embed_file.bytes = file_bytes
@@ -57,41 +64,62 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
 		}
 		node.embed_file.len = file_bytes.len
 	}
+	ef_idx := node.embed_file.hash()
+	g.write('_v_embed_file_metadata($ef_idx)')
+	g.file.embedded_files << node.embed_file
+	$if trace_embed_file ? {
+		eprintln('> gen_embed_file_init => _v_embed_file_metadata(${ef_idx:-25}) | ${node.embed_file.apath:-50} | compression: $node.embed_file.compression_type | len: $node.embed_file.len')
+	}
+}
 
-	g.writeln('(v__embed_file__EmbedFileData){')
+// gen_embedded_metadata embeds all of the deduplicated metadata in g.embedded_files, into the V target executable,
-	g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},')
+// into a single generated function _v_embed_file_metadata, that accepts a hash of the absolute path of the embedded
+// files.
+fn (mut g Gen) gen_embedded_metadata() {
+	g.embedded_data.writeln('v__embed_file__EmbedFileData _v_embed_file_metadata(u64 ef_hash) {')
+	g.embedded_data.writeln('\tv__embed_file__EmbedFileData res;')
+	g.embedded_data.writeln('\tmemset(&res, 0, sizeof(res));')
+	g.embedded_data.writeln('\tswitch(ef_hash) {')
+	for emfile in g.embedded_files {
+		ef_idx := emfile.hash()
+		g.embedded_data.writeln('\t\tcase $ef_idx: {')
+		g.embedded_data.writeln('\t\t\tres.path = ${ctoslit(emfile.rpath)};')
 		if g.embed_file_is_prod_mode() {
 			// apath is not needed in production and may leak information
-		g.writeln('\t\t.apath = ${ctoslit('')},')
+			g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit('')};')
 		} else {
-		g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},')
+			g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit(emfile.apath)};')
 		}
 		if g.embed_file_is_prod_mode() {
 			// use function generated in Gen.gen_embedded_data()
-		if node.embed_file.is_compressed {
+			if emfile.is_compressed {
-			g.writeln('\t\t.compression_type = ${ctoslit(node.embed_file.compression_type)},')
+				g.embedded_data.writeln('\t\t\tres.compression_type = ${ctoslit(emfile.compression_type)};')
-			g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,')
+				g.embedded_data.writeln('\t\t\tres.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;')
-			g.writeln('\t\t.uncompressed = NULL,')
+				g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;')
 			} else {
-			g.writeln('\t\t.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,')
+				g.embedded_data.writeln('\t\t\tres.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;')
 			}
 		} else {
-		g.writeln('\t\t.uncompressed = NULL,')
+			g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;')
 		}
-	g.writeln('\t\t.free_compressed = 0,')
+		g.embedded_data.writeln('\t\t\tres.free_compressed = 0;')
-	g.writeln('\t\t.free_uncompressed = 0,')
+		g.embedded_data.writeln('\t\t\tres.free_uncompressed = 0;')
 		if g.embed_file_is_prod_mode() {
-		g.writeln('\t\t.len = $node.embed_file.len')
+			g.embedded_data.writeln('\t\t\tres.len = $emfile.len;')
 		} else {
-		file_size := os.file_size(node.embed_file.apath)
+			file_size := os.file_size(emfile.apath)
 			if file_size > 5242880 {
 				eprintln('Warning: embedding of files >= ~5MB is currently not supported')
 			}
-		g.writeln('\t\t.len = $file_size')
+			g.embedded_data.writeln('\t\t\tres.len = $file_size;')
 		}
-	g.writeln('} // \$embed_file("$node.embed_file.apath")')
+		g.embedded_data.writeln('\t\t\tbreak;')
-
+		g.embedded_data.writeln('\t\t} // case $ef_idx')
-	g.file.embedded_files << node.embed_file
+	}
+	g.embedded_data.writeln('\t\tdefault: _v_panic(_SLIT("unknown embed file"));')
+	g.embedded_data.writeln('\t} // switch')
+	g.embedded_data.writeln('\treturn res;')
+	g.embedded_data.writeln('}')
 }
 
 // gen_embedded_data embeds data into the V target executable.

vlib/v/gen/c/testdata/embed.c.must_have

@@ -13,7 +13,7 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
 string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
 v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
 
-v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){
+v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
-.path = _SLIT("embed.vv"),
+res.path = _SLIT("embed.vv");
 
 // Initializations for module v.embed_file :

vlib/v/gen/c/testdata/embed_with_prod.c.must_have

@@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
 string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
 v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
 
-v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){
+v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
-.path = _SLIT("embed.vv"),
+res.path = _SLIT("embed.vv");
-.apath = _SLIT(""),
+res.apath = _SLIT("");
-.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data,
+res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data;
 
 // Initializations for module v.embed_file :

vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have

@@ -6,6 +6,6 @@ VV_LOCAL_SYMBOL void v__preludes__embed_file__zlib__init(void);
 VV_LOCAL_SYMBOL Option_Array_byte v__preludes__embed_file__zlib__ZLibDecoder_decompress(v__preludes__embed_file__zlib__ZLibDecoder _d1, Array_byte data) {
 = compress__zlib__decompress(data);
 
-.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data,
+res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data;
-.compression_type = _SLIT("zlib")
+res.compression_type = _SLIT("zlib");
-.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data,
+res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data;

vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have

@@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
 string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
 v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
 
-v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){
+v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
-.path = _SLIT("embed.vv"),
+res.path = _SLIT("embed.vv");
-.apath = _SLIT(""),
+res.apath = _SLIT("");
-.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data,
+res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data;
 
 // Initializations for module v.embed_file :

vlib/v/parser/expr.v

@@ -78,7 +78,8 @@ pub fn (mut p Parser) check_expr(precedence int) ?ast.Expr {
 		.dollar {
 			match p.peek_tok.kind {
 				.name {
-					return p.comptime_call()
+					node = p.comptime_call()
+					p.is_stmt_ident = is_stmt_ident
 				}
 				.key_if {
 					return p.if_expr(true)