diff --git a/vlib/embed/embed.v b/vlib/embed/embed.v new file mode 100644 index 0000000000..c9e9596ae4 --- /dev/null +++ b/vlib/embed/embed.v @@ -0,0 +1,61 @@ +module embed + +import os + +// https://github.com/vlang/rfcs/blob/master/embedding_resources.md +// EmbeddedData encapsulates functionality for the `$embed_file()` compile time call. +pub struct EmbeddedData { + path string + apath string +mut: + compressed byteptr + uncompressed byteptr + free_compressed bool + free_uncompressed bool +pub: + len int +} + +pub fn (ed EmbeddedData) str() string { + return 'embed.EmbeddedData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }' +} + +[unsafe] +pub fn (mut ed EmbeddedData) free() { + unsafe { + ed.path.free() + ed.apath.free() + if ed.free_compressed { + free(ed.compressed) + } + if ed.free_uncompressed { + free(ed.uncompressed) + } + } +} + +pub fn (mut ed EmbeddedData) data() byteptr { + if !isnil(ed.uncompressed) { + return ed.uncompressed + } else { + if isnil(ed.uncompressed) && !isnil(ed.compressed) { + // TODO implement uncompression + // See also C Gen.gen_embedded_data() where the compression should occur. + ed.uncompressed = ed.compressed + } else { + mut path := os.resource_abs_path(ed.path) + if !os.is_file(path) { + path = ed.apath + if !os.is_file(path) { + panic('EmbeddedData error: files "$ed.path" and "$ed.apath" do not exist') + } + } + bytes := os.read_bytes(path) or { + panic('EmbeddedData error: "$path" could not be read: $err') + } + ed.uncompressed = bytes.data + ed.free_uncompressed = true + } + } + return ed.uncompressed +} diff --git a/vlib/v/ast/ast.v b/vlib/v/ast/ast.v index e4621d8b9d..cf8aa07178 100644 --- a/vlib/v/ast/ast.v +++ b/vlib/v/ast/ast.v @@ -458,6 +458,12 @@ pub mut: end_comments []Comment } +pub struct EmbeddedFile { +pub: + rpath string // used in the source code, as an ID/key to the embed + apath string // absolute path during compilation to the resource +} + // Each V source file is represented by one ast.File structure. // When the V compiler runs, the parser will fill an []ast.File. // That array is then passed to V's checker. @@ -468,9 +474,10 @@ pub: global_scope &Scope pub mut: scope &Scope - stmts []Stmt // all the statements in the source file - imports []Import // all the imports - auto_imports []string // imports that were implicitely added + stmts []Stmt // all the statements in the source file + imports []Import // all the imports + auto_imports []string // imports that were implicitely added + embedded_files []EmbeddedFile // list of files to embed in the binary imported_symbols map[string]string // used for `import {symbol}`, it maps symbol => module.symbol errors []errors.Error // all the checker errors in the file warnings []errors.Warning // all the checker warings in the file @@ -1088,6 +1095,8 @@ pub: is_vweb bool vweb_tmpl File args_var string + is_embed bool + embed_file EmbeddedFile pub mut: sym table.TypeSymbol } diff --git a/vlib/v/checker/checker.v b/vlib/v/checker/checker.v index 8accfc7f18..489fa15e11 100644 --- a/vlib/v/checker/checker.v +++ b/vlib/v/checker/checker.v @@ -3248,6 +3248,10 @@ pub fn (mut c Checker) expr(node ast.Expr) table.Type { } ast.ComptimeCall { node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left))) + if node.is_embed { + c.file.embedded_files << node.embed_file + return c.table.find_type_idx('embed.EmbeddedData') + } if node.is_vweb { // TODO assoc parser bug pref := *c.pref diff --git a/vlib/v/fmt/fmt.v b/vlib/v/fmt/fmt.v index 555fffe716..ab77dd7607 100644 --- a/vlib/v/fmt/fmt.v +++ b/vlib/v/fmt/fmt.v @@ -970,12 +970,16 @@ pub fn (mut f Fmt) expr(node ast.Expr) { f.write("\$tmpl('$node.args_var')") } } else { - method_expr := if node.has_parens { - '(${node.method_name}($node.args_var))' + if node.is_embed { + f.write("\$embed_file('$node.embed_file.rpath')") } else { - '${node.method_name}($node.args_var)' + method_expr := if node.has_parens { + '(${node.method_name}($node.args_var))' + } else { + '${node.method_name}($node.args_var)' + } + f.write('${node.left}.$$method_expr') } - f.write('${node.left}.$$method_expr') } } ast.ComptimeSelector { diff --git a/vlib/v/fmt/tests/embed_file_keep.vv b/vlib/v/fmt/tests/embed_file_keep.vv new file mode 100644 index 0000000000..2234bd79d3 --- /dev/null +++ b/vlib/v/fmt/tests/embed_file_keep.vv @@ -0,0 +1,8 @@ +fn main() { + mut the_png := $embed_file('v.png') + println(the_png) + content := the_png.data() + eprintln('content: ${ptr_str(content)}') + eprintln(unsafe { the_png.data().vbytes(the_png.len) }.hex()) + println(the_png) +} diff --git a/vlib/v/gen/cgen.v b/vlib/v/gen/cgen.v index f05214367a..a5c5a4af76 100644 --- a/vlib/v/gen/cgen.v +++ b/vlib/v/gen/cgen.v @@ -44,6 +44,7 @@ mut: comptime_defines strings.Builder // custom defines, given by -d/-define flags on the CLI pcs_declarations strings.Builder // -prof profile counter declarations for each function hotcode_definitions strings.Builder // -live declarations & functions + embedded_data strings.Builder // data to embed in the executable/binary shared_types strings.Builder // shared/lock types channel_definitions strings.Builder // channel related code options_typedefs strings.Builder // Option typedefs @@ -97,6 +98,7 @@ mut: pcs []ProfileCounterMeta // -prof profile counter fn_names => fn counter name is_builtin_mod bool hotcode_fn_names []string + embedded_files []ast.EmbeddedFile // cur_fn ast.FnDecl cur_generic_type table.Type // `int`, `string`, etc in `foo()` sql_i int @@ -176,6 +178,7 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string comptime_defines: strings.new_builder(100) pcs_declarations: strings.new_builder(100) hotcode_definitions: strings.new_builder(100) + embedded_data: strings.new_builder(1000) options_typedefs: strings.new_builder(100) options: strings.new_builder(100) shared_types: strings.new_builder(100) @@ -228,6 +231,14 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string tests_inited = true } g.stmts(file.stmts) + // Transfer embedded files + if file.embedded_files.len > 0 { + for path in file.embedded_files { + if path !in g.embedded_files { + g.embedded_files << path + } + } + } g.timers.show('cgen_file $file.path') } g.timers.start('cgen common') @@ -299,6 +310,10 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string b.writeln('\n// V hotcode definitions:') b.write(g.hotcode_definitions.str()) } + if g.embedded_data.len > 0 { + b.writeln('\n// V embedded data:') + b.write(g.embedded_data.str()) + } if g.options_typedefs.len > 0 { b.writeln('\n// V option typedefs:') b.write(g.options_typedefs.str()) @@ -421,6 +436,9 @@ pub fn (mut g Gen) finish() { if g.pref.is_livemain || g.pref.is_liveshared { g.generate_hotcode_reloader_code() } + if g.pref.is_prod && g.embedded_files.len > 0 { + g.gen_embedded_data() + } if g.pref.is_test { g.gen_c_main_for_tests() } else { diff --git a/vlib/v/gen/comptime.v b/vlib/v/gen/comptime.v index c977cbfd96..7590190f02 100644 --- a/vlib/v/gen/comptime.v +++ b/vlib/v/gen/comptime.v @@ -28,6 +28,10 @@ fn (mut g Gen) comptime_selector(node ast.ComptimeSelector) { } fn (mut g Gen) comptime_call(node ast.ComptimeCall) { + if node.is_embed { + g.gen_embed_file_init(node) + return + } if node.is_vweb { is_html := node.method_name == 'html' for stmt in node.vweb_tmpl.stmts { diff --git a/vlib/v/gen/embed.v b/vlib/v/gen/embed.v new file mode 100644 index 0000000000..dd6ea9e22c --- /dev/null +++ b/vlib/v/gen/embed.v @@ -0,0 +1,75 @@ +module gen + +import os +import v.ast + +// gen_embed_file_struct generates C code for `$embed_file('...')` calls. +fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) { + g.writeln('(embed__EmbeddedData){') + g.writeln('\t.path = ${ctoslit(node.embed_file.rpath)},') + g.writeln('\t.apath = ${ctoslit(node.embed_file.apath)},') + file_size := os.file_size(node.embed_file.apath) + if file_size > 5242880 { + eprintln('Warning: embedding of files >= ~5MB is currently not supported') + } + if g.pref.is_prod { + // Use function generated in Gen.gen_embedded_data() + g.writeln('\t.compressed = _v_embed_locate_data(${ctoslit(node.embed_file.apath)}),') + } + g.writeln('\t.len = $file_size') + g.writeln('} // $' + 'embed_file("$node.embed_file.apath")') +} + +// gen_embedded_data embeds data into the V target executable. +fn (mut g Gen) gen_embedded_data() { + /* + TODO implement compression. + See also the vlib/embed module where decompression should occur. + */ + /* + TODO implement support for large files - right now the setup has problems + // with even just 10 - 50 MB files - the problem is both in V and C compilers. + // maybe we need to write to separate files or have an external tool for large files + // like the `rcc` tool in Qt? + */ + for i, emfile in g.embedded_files { + fbytes := os.read_bytes(emfile.apath) or { panic('Error while embedding file: $err') } + g.embedded_data.write('static const unsigned char _v_embed_blob_$i[$fbytes.len] = {\n ') + for j := 0; j < fbytes.len; j++ { + b := fbytes[j].hex() + if j < fbytes.len - 1 { + g.embedded_data.write('0x$b,') + } else { + g.embedded_data.write('0x$b') + } + if 0 == ((j + 1) % 16) { + g.embedded_data.write('\n ') + } + } + g.embedded_data.writeln('\n};') + } + g.embedded_data.writeln('') + g.embedded_data.writeln('const struct _v_embed {') + g.embedded_data.writeln('\tstring id;') + g.embedded_data.writeln('\tbyteptr data;') + g.embedded_data.writeln('}') + g.embedded_data.writeln('_v_embedded_data[] = {') + for i, emfile in g.embedded_files { + g.embedded_data.writeln('\t{${ctoslit(emfile.rpath)}, _v_embed_blob_$i},') + } + g.embedded_data.writeln('\t{_SLIT(""), NULL}') + g.embedded_data.writeln('};') + // See `vlib/v/gen/comptime.v` -> Gen.comptime_call_embed_file(), where this is called at runtime. + // Generate function to locate the data. + g.embedded_data.writeln(' +// function to locate embedded data by a vstring +byteptr _v_embed_locate_data(string id) { + const struct _v_embed *ve; + for (ve = _v_embedded_data; !string_eq(ve->id, _SLIT("")) && ve->data != NULL; ve++) { + if (string_eq(ve->id, id)) { + return (byteptr) ve->data; + } + } + return NULL; +}') +} diff --git a/vlib/v/parser/comptime.v b/vlib/v/parser/comptime.v index c5865c618b..6ee182e5af 100644 --- a/vlib/v/parser/comptime.v +++ b/vlib/v/parser/comptime.v @@ -10,6 +10,10 @@ import v.table import v.token import vweb.tmpl +const ( + supported_comptime_calls = ['html', 'tmpl', 'embed_file'] +) + // // #include, #flag, #v fn (mut p Parser) hash() ast.HashStmt { mut pos := p.prev_tok.position() @@ -37,9 +41,9 @@ fn (mut p Parser) hash() ast.HashStmt { } } -fn (mut p Parser) vweb() ast.ComptimeCall { +fn (mut p Parser) comp_call() ast.ComptimeCall { p.check(.dollar) - error_msg := 'only `\$tmpl()` and `\$vweb.html()` comptime functions are supported right now' + error_msg := 'only `\$tmpl()`, `\$embed_file()` and `\$vweb.html()` comptime functions are supported right now' if p.peek_tok.kind == .dot { n := p.check_name() // skip `vweb.html()` TODO if n != 'vweb' { @@ -49,17 +53,57 @@ fn (mut p Parser) vweb() ast.ComptimeCall { p.check(.dot) } n := p.check_name() // (.name) - if n != 'html' && n != 'tmpl' { + if n !in supported_comptime_calls { p.error(error_msg) return ast.ComptimeCall{} } + is_embed_file := n == 'embed_file' is_html := n == 'html' p.check(.lpar) + spos := p.tok.position() s := if is_html { '' } else { p.tok.lit } if !is_html { p.check(.string) } p.check(.rpar) + // + if is_embed_file { + mut epath := s + // Validate that the epath exists, and that it is actually a file. + if epath == '' { + p.error_with_pos('please supply a valid relative or absolute file path to the file to embed', + spos) + return ast.ComptimeCall{} + } + if !p.pref.is_fmt { + abs_path := os.real_path(epath) + // check absolute path first + if !os.exists(abs_path) { + // ... look relative to the source file: + epath = os.real_path(os.join_path(os.dir(p.file_name), epath)) + if !os.exists(epath) { + p.error_with_pos('"$epath" does not exist so it cannot be embedded', + spos) + return ast.ComptimeCall{} + } + if !os.is_file(epath) { + p.error_with_pos('"$epath" is not a file so it cannot be embedded', + spos) + return ast.ComptimeCall{} + } + } else { + epath = abs_path + } + } + p.register_auto_import('embed') + return ast.ComptimeCall{ + is_embed: true + embed_file: ast.EmbeddedFile{ + rpath: s + apath: epath + } + } + } // Compile vweb html template to V code, parse that V code and embed the resulting V function // that returns an html string. fn_path := p.cur_fn_name.split('_') @@ -71,6 +115,7 @@ fn (mut p Parser) vweb() ast.ComptimeCall { if !is_html { path = tmpl_path } + eprintln('>>> is_embed_file: $is_embed_file | is_html: $is_html | s: $s | n: $n | path: $path') if !os.exists(path) { // can be in `templates/` if is_html { diff --git a/vlib/v/parser/parser.v b/vlib/v/parser/parser.v index 293cd8ff12..dd1480aafb 100644 --- a/vlib/v/parser/parser.v +++ b/vlib/v/parser/parser.v @@ -696,7 +696,7 @@ pub fn (mut p Parser) stmt(is_top_level bool) ast.Stmt { } .name { return ast.ExprStmt{ - expr: p.vweb() + expr: p.comp_call() } } else { diff --git a/vlib/v/parser/pratt.v b/vlib/v/parser/pratt.v index 02069735e9..eb1821c917 100644 --- a/vlib/v/parser/pratt.v +++ b/vlib/v/parser/pratt.v @@ -56,7 +56,7 @@ pub fn (mut p Parser) expr(precedence int) ast.Expr { .dollar { match p.peek_tok.kind { .name { - return p.vweb() + return p.comp_call() } .key_if { return p.if_expr(true) diff --git a/vlib/v/tests/embed_file/embed_file_test.v b/vlib/v/tests/embed_file/embed_file_test.v new file mode 100644 index 0000000000..2af4459ea4 --- /dev/null +++ b/vlib/v/tests/embed_file/embed_file_test.v @@ -0,0 +1,25 @@ +const const_file = $embed_file('v.png') + +fn test_const_embed_file() { + mut file := const_file + eprintln('file: $file') + assert file.len == 603 + fdata := file.data() + eprintln('file after .data() call: $file') + assert file.len == 603 + unsafe { + assert fdata.vbytes(4) == [byte(0x89), `P`, `N`, `G`] + } +} + +fn test_embed_file() { + mut file := $embed_file('v.png') + eprintln('file: $file') + assert file.len == 603 + fdata := file.data() + eprintln('file after .data() call: $file') + assert file.len == 603 + unsafe { + assert fdata.vbytes(4) == [byte(0x89), `P`, `N`, `G`] + } +} diff --git a/vlib/v/tests/embed_file/v.png b/vlib/v/tests/embed_file/v.png new file mode 100644 index 0000000000..8739e46772 Binary files /dev/null and b/vlib/v/tests/embed_file/v.png differ