all: add support for the `x := $embed_file('v.png')` compile time call (#8048)

vlib/embed/embed.v

@@ -0,0 +1,61 @@
+module embed
+
+import os
+
+// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
+// EmbeddedData encapsulates functionality for the `$embed_file()` compile time call.
+pub struct EmbeddedData {
+	path  string
+	apath string
+mut:
+	compressed        byteptr
+	uncompressed      byteptr
+	free_compressed   bool
+	free_uncompressed bool
+pub:
+	len int
+}
+
+pub fn (ed EmbeddedData) str() string {
+	return 'embed.EmbeddedData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
+}
+
+[unsafe]
+pub fn (mut ed EmbeddedData) free() {
+	unsafe {
+		ed.path.free()
+		ed.apath.free()
+		if ed.free_compressed {
+			free(ed.compressed)
+		}
+		if ed.free_uncompressed {
+			free(ed.uncompressed)
+		}
+	}
+}
+
+pub fn (mut ed EmbeddedData) data() byteptr {
+	if !isnil(ed.uncompressed) {
+		return ed.uncompressed
+	} else {
+		if isnil(ed.uncompressed) && !isnil(ed.compressed) {
+			// TODO implement uncompression
+			// See also C Gen.gen_embedded_data() where the compression should occur.
+			ed.uncompressed = ed.compressed
+		} else {
+			mut path := os.resource_abs_path(ed.path)
+			if !os.is_file(path) {
+				path = ed.apath
+				if !os.is_file(path) {
+					panic('EmbeddedData error: files "$ed.path" and "$ed.apath" do not exist')
+				}
+			}
+			bytes := os.read_bytes(path) or {
+				panic('EmbeddedData error: "$path" could not be read: $err')
+			}
+			ed.uncompressed = bytes.data
+			ed.free_uncompressed = true
+		}
+	}
+	return ed.uncompressed
+}

vlib/v/ast/ast.v

@@ -458,6 +458,12 @@ pub mut:
 	end_comments []Comment
 }
 
+pub struct EmbeddedFile {
+pub:
+	rpath string // used in the source code, as an ID/key to the embed
+	apath string // absolute path during compilation to the resource
+}
+
 // Each V source file is represented by one ast.File structure.
 // When the V compiler runs, the parser will fill an []ast.File.
 // That array is then passed to V's checker.
@@ -471,6 +477,7 @@ pub mut:
 	stmts            []Stmt            // all the statements in the source file
 	imports          []Import          // all the imports
 	auto_imports     []string          // imports that were implicitely added
+	embedded_files   []EmbeddedFile    // list of files to embed in the binary
 	imported_symbols map[string]string // used for `import {symbol}`, it maps symbol => module.symbol
 	errors           []errors.Error    // all the checker errors in the file
 	warnings         []errors.Warning  // all the checker warings in the file
@@ -1088,6 +1095,8 @@ pub:
 	is_vweb     bool
 	vweb_tmpl   File
 	args_var    string
+	is_embed    bool
+	embed_file  EmbeddedFile
 pub mut:
 	sym table.TypeSymbol
 }

vlib/v/checker/checker.v

@@ -3248,6 +3248,10 @@ pub fn (mut c Checker) expr(node ast.Expr) table.Type {
 		}
 		ast.ComptimeCall {
 			node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left)))
+			if node.is_embed {
+				c.file.embedded_files << node.embed_file
+				return c.table.find_type_idx('embed.EmbeddedData')
+			}
 			if node.is_vweb {
 				// TODO assoc parser bug
 				pref := *c.pref

vlib/v/fmt/fmt.v

@@ -969,6 +969,9 @@ pub fn (mut f Fmt) expr(node ast.Expr) {
 				} else {
 					f.write("\$tmpl('$node.args_var')")
 				}
+			} else {
+				if node.is_embed {
+					f.write("\$embed_file('$node.embed_file.rpath')")
 				} else {
 					method_expr := if node.has_parens {
 						'(${node.method_name}($node.args_var))'
@@ -978,6 +981,7 @@ pub fn (mut f Fmt) expr(node ast.Expr) {
 					f.write('${node.left}.$$method_expr')
 				}
 			}
+		}
 		ast.ComptimeSelector {
 			field_expr := if node.has_parens { '($node.field_expr)' } else { node.field_expr.str() }
 			f.write('${node.left}.$$field_expr')

vlib/v/fmt/tests/embed_file_keep.vv

@@ -0,0 +1,8 @@
+fn main() {
+	mut the_png := $embed_file('v.png')
+	println(the_png)
+	content := the_png.data()
+	eprintln('content: ${ptr_str(content)}')
+	eprintln(unsafe { the_png.data().vbytes(the_png.len) }.hex())
+	println(the_png)
+}

vlib/v/gen/cgen.v

@@ -44,6 +44,7 @@ mut:
 	comptime_defines    strings.Builder // custom defines, given by -d/-define flags on the CLI
 	pcs_declarations    strings.Builder // -prof profile counter declarations for each function
 	hotcode_definitions strings.Builder // -live declarations & functions
+	embedded_data       strings.Builder // data to embed in the executable/binary
 	shared_types        strings.Builder // shared/lock types
 	channel_definitions strings.Builder // channel related code
 	options_typedefs    strings.Builder // Option typedefs
@@ -97,6 +98,7 @@ mut:
 	pcs                   []ProfileCounterMeta // -prof profile counter fn_names => fn counter name
 	is_builtin_mod        bool
 	hotcode_fn_names      []string
+	embedded_files        []ast.EmbeddedFile
 	// cur_fn               ast.FnDecl
 	cur_generic_type table.Type // `int`, `string`, etc in `foo<T>()`
 	sql_i            int
@@ -176,6 +178,7 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string
 		comptime_defines: strings.new_builder(100)
 		pcs_declarations: strings.new_builder(100)
 		hotcode_definitions: strings.new_builder(100)
+		embedded_data: strings.new_builder(1000)
 		options_typedefs: strings.new_builder(100)
 		options: strings.new_builder(100)
 		shared_types: strings.new_builder(100)
@@ -228,6 +231,14 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string
 			tests_inited = true
 		}
 		g.stmts(file.stmts)
+		// Transfer embedded files
+		if file.embedded_files.len > 0 {
+			for path in file.embedded_files {
+				if path !in g.embedded_files {
+					g.embedded_files << path
+				}
+			}
+		}
 		g.timers.show('cgen_file $file.path')
 	}
 	g.timers.start('cgen common')
@@ -299,6 +310,10 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string
 		b.writeln('\n// V hotcode definitions:')
 		b.write(g.hotcode_definitions.str())
 	}
+	if g.embedded_data.len > 0 {
+		b.writeln('\n// V embedded data:')
+		b.write(g.embedded_data.str())
+	}
 	if g.options_typedefs.len > 0 {
 		b.writeln('\n// V option typedefs:')
 		b.write(g.options_typedefs.str())
@@ -421,6 +436,9 @@ pub fn (mut g Gen) finish() {
 	if g.pref.is_livemain || g.pref.is_liveshared {
 		g.generate_hotcode_reloader_code()
 	}
+	if g.pref.is_prod && g.embedded_files.len > 0 {
+		g.gen_embedded_data()
+	}
 	if g.pref.is_test {
 		g.gen_c_main_for_tests()
 	} else {

vlib/v/gen/comptime.v

@@ -28,6 +28,10 @@ fn (mut g Gen) comptime_selector(node ast.ComptimeSelector) {
 }
 
 fn (mut g Gen) comptime_call(node ast.ComptimeCall) {
+	if node.is_embed {
+		g.gen_embed_file_init(node)
+		return
+	}
 	if node.is_vweb {
 		is_html := node.method_name == 'html'
 		for stmt in node.vweb_tmpl.stmts {

vlib/v/gen/embed.v

@@ -0,0 +1,75 @@
+module gen
+
+import os
+import v.ast
+
+// gen_embed_file_struct generates C code for `$embed_file('...')` calls.
+fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) {
+	g.writeln('(embed__EmbeddedData){')
+	g.writeln('\t.path = ${ctoslit(node.embed_file.rpath)},')
+	g.writeln('\t.apath = ${ctoslit(node.embed_file.apath)},')
+	file_size := os.file_size(node.embed_file.apath)
+	if file_size > 5242880 {
+		eprintln('Warning: embedding of files >= ~5MB is currently not supported')
+	}
+	if g.pref.is_prod {
+		// Use function generated in Gen.gen_embedded_data()
+		g.writeln('\t.compressed = _v_embed_locate_data(${ctoslit(node.embed_file.apath)}),')
+	}
+	g.writeln('\t.len = $file_size')
+	g.writeln('} // $' + 'embed_file("$node.embed_file.apath")')
+}
+
+// gen_embedded_data embeds data into the V target executable.
+fn (mut g Gen) gen_embedded_data() {
+	/*
+	TODO implement compression.
+	See also the vlib/embed module where decompression should occur.
+	*/
+	/*
+	TODO implement support for large files - right now the setup has problems
+	// with even just 10 - 50 MB files - the problem is both in V and C compilers.
+	// maybe we need to write to separate files or have an external tool for large files
+	// like the `rcc` tool in Qt?
+	*/
+	for i, emfile in g.embedded_files {
+		fbytes := os.read_bytes(emfile.apath) or { panic('Error while embedding file: $err') }
+		g.embedded_data.write('static const unsigned char _v_embed_blob_$i[$fbytes.len] = {\n    ')
+		for j := 0; j < fbytes.len; j++ {
+			b := fbytes[j].hex()
+			if j < fbytes.len - 1 {
+				g.embedded_data.write('0x$b,')
+			} else {
+				g.embedded_data.write('0x$b')
+			}
+			if 0 == ((j + 1) % 16) {
+				g.embedded_data.write('\n    ')
+			}
+		}
+		g.embedded_data.writeln('\n};')
+	}
+	g.embedded_data.writeln('')
+	g.embedded_data.writeln('const struct _v_embed {')
+	g.embedded_data.writeln('\tstring id;')
+	g.embedded_data.writeln('\tbyteptr data;')
+	g.embedded_data.writeln('}')
+	g.embedded_data.writeln('_v_embedded_data[] = {')
+	for i, emfile in g.embedded_files {
+		g.embedded_data.writeln('\t{${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
+	}
+	g.embedded_data.writeln('\t{_SLIT(""), NULL}')
+	g.embedded_data.writeln('};')
+	// See `vlib/v/gen/comptime.v` -> Gen.comptime_call_embed_file(), where this is called at runtime.
+	// Generate function to locate the data.
+	g.embedded_data.writeln('
+// function to locate embedded data by a vstring
+byteptr _v_embed_locate_data(string id) {
+	const struct _v_embed *ve;
+	for (ve = _v_embedded_data; !string_eq(ve->id, _SLIT("")) && ve->data != NULL; ve++) {
+		if (string_eq(ve->id, id)) {
+			return (byteptr) ve->data;
+		}
+	}
+	return NULL;
+}')
+}

vlib/v/parser/comptime.v

@@ -10,6 +10,10 @@ import v.table
 import v.token
 import vweb.tmpl
 
+const (
+	supported_comptime_calls = ['html', 'tmpl', 'embed_file']
+)
+
 // // #include, #flag, #v
 fn (mut p Parser) hash() ast.HashStmt {
 	mut pos := p.prev_tok.position()
@@ -37,9 +41,9 @@ fn (mut p Parser) hash() ast.HashStmt {
 	}
 }
 
-fn (mut p Parser) vweb() ast.ComptimeCall {
+fn (mut p Parser) comp_call() ast.ComptimeCall {
 	p.check(.dollar)
-	error_msg := 'only `\$tmpl()` and `\$vweb.html()` comptime functions are supported right now'
+	error_msg := 'only `\$tmpl()`, `\$embed_file()` and `\$vweb.html()` comptime functions are supported right now'
 	if p.peek_tok.kind == .dot {
 		n := p.check_name() // skip `vweb.html()` TODO
 		if n != 'vweb' {
@@ -49,17 +53,57 @@ fn (mut p Parser) vweb() ast.ComptimeCall {
 		p.check(.dot)
 	}
 	n := p.check_name() // (.name)
-	if n != 'html' && n != 'tmpl' {
+	if n !in supported_comptime_calls {
 		p.error(error_msg)
 		return ast.ComptimeCall{}
 	}
+	is_embed_file := n == 'embed_file'
 	is_html := n == 'html'
 	p.check(.lpar)
+	spos := p.tok.position()
 	s := if is_html { '' } else { p.tok.lit }
 	if !is_html {
 		p.check(.string)
 	}
 	p.check(.rpar)
+	//
+	if is_embed_file {
+		mut epath := s
+		// Validate that the epath exists, and that it is actually a file.
+		if epath == '' {
+			p.error_with_pos('please supply a valid relative or absolute file path to the file to embed',
+				spos)
+			return ast.ComptimeCall{}
+		}
+		if !p.pref.is_fmt {
+			abs_path := os.real_path(epath)
+			// check absolute path first
+			if !os.exists(abs_path) {
+				// ... look relative to the source file:
+				epath = os.real_path(os.join_path(os.dir(p.file_name), epath))
+				if !os.exists(epath) {
+					p.error_with_pos('"$epath" does not exist so it cannot be embedded',
+						spos)
+					return ast.ComptimeCall{}
+				}
+				if !os.is_file(epath) {
+					p.error_with_pos('"$epath" is not a file so it cannot be embedded',
+						spos)
+					return ast.ComptimeCall{}
+				}
+			} else {
+				epath = abs_path
+			}
+		}
+		p.register_auto_import('embed')
+		return ast.ComptimeCall{
+			is_embed: true
+			embed_file: ast.EmbeddedFile{
+				rpath: s
+				apath: epath
+			}
+		}
+	}
 	// Compile vweb html template to V code, parse that V code and embed the resulting V function
 	// that returns an html string.
 	fn_path := p.cur_fn_name.split('_')
@@ -71,6 +115,7 @@ fn (mut p Parser) vweb() ast.ComptimeCall {
 	if !is_html {
 		path = tmpl_path
 	}
+	eprintln('>>> is_embed_file: $is_embed_file | is_html: $is_html | s: $s | n: $n | path: $path')
 	if !os.exists(path) {
 		// can be in `templates/`
 		if is_html {

vlib/v/parser/parser.v

@@ -696,7 +696,7 @@ pub fn (mut p Parser) stmt(is_top_level bool) ast.Stmt {
 				}
 				.name {
 					return ast.ExprStmt{
-						expr: p.vweb()
+						expr: p.comp_call()
 					}
 				}
 				else {

vlib/v/parser/pratt.v

@@ -56,7 +56,7 @@ pub fn (mut p Parser) expr(precedence int) ast.Expr {
 		.dollar {
 			match p.peek_tok.kind {
 				.name {
-					return p.vweb()
+					return p.comp_call()
 				}
 				.key_if {
 					return p.if_expr(true)

vlib/v/tests/embed_file/embed_file_test.v

@@ -0,0 +1,25 @@
+const const_file = $embed_file('v.png')
+
+fn test_const_embed_file() {
+	mut file := const_file
+	eprintln('file: $file')
+	assert file.len == 603
+	fdata := file.data()
+	eprintln('file after .data() call: $file')
+	assert file.len == 603
+	unsafe {
+		assert fdata.vbytes(4) == [byte(0x89), `P`, `N`, `G`]
+	}
+}
+
+fn test_embed_file() {
+	mut file := $embed_file('v.png')
+	eprintln('file: $file')
+	assert file.len == 603
+	fdata := file.data()
+	eprintln('file after .data() call: $file')
+	assert file.len == 603
+	unsafe {
+		assert fdata.vbytes(4) == [byte(0x89), `P`, `N`, `G`]
+	}
+}