all: add support for the `x := $embed_file('v.png')` compile time call (#8048)

pull/8108/head
Larpon 2021-01-14 15:20:11 +01:00 committed by GitHub
parent 9003ea7ca3
commit f73500f2fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 265 additions and 12 deletions

61
vlib/embed/embed.v 100644
View File

@ -0,0 +1,61 @@
module embed
import os
// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
// EmbeddedData encapsulates functionality for the `$embed_file()` compile time call.
pub struct EmbeddedData {
path string
apath string
mut:
compressed byteptr
uncompressed byteptr
free_compressed bool
free_uncompressed bool
pub:
len int
}
pub fn (ed EmbeddedData) str() string {
return 'embed.EmbeddedData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
}
[unsafe]
pub fn (mut ed EmbeddedData) free() {
unsafe {
ed.path.free()
ed.apath.free()
if ed.free_compressed {
free(ed.compressed)
}
if ed.free_uncompressed {
free(ed.uncompressed)
}
}
}
pub fn (mut ed EmbeddedData) data() byteptr {
if !isnil(ed.uncompressed) {
return ed.uncompressed
} else {
if isnil(ed.uncompressed) && !isnil(ed.compressed) {
// TODO implement uncompression
// See also C Gen.gen_embedded_data() where the compression should occur.
ed.uncompressed = ed.compressed
} else {
mut path := os.resource_abs_path(ed.path)
if !os.is_file(path) {
path = ed.apath
if !os.is_file(path) {
panic('EmbeddedData error: files "$ed.path" and "$ed.apath" do not exist')
}
}
bytes := os.read_bytes(path) or {
panic('EmbeddedData error: "$path" could not be read: $err')
}
ed.uncompressed = bytes.data
ed.free_uncompressed = true
}
}
return ed.uncompressed
}

View File

@ -458,6 +458,12 @@ pub mut:
end_comments []Comment end_comments []Comment
} }
pub struct EmbeddedFile {
pub:
rpath string // used in the source code, as an ID/key to the embed
apath string // absolute path during compilation to the resource
}
// Each V source file is represented by one ast.File structure. // Each V source file is represented by one ast.File structure.
// When the V compiler runs, the parser will fill an []ast.File. // When the V compiler runs, the parser will fill an []ast.File.
// That array is then passed to V's checker. // That array is then passed to V's checker.
@ -471,6 +477,7 @@ pub mut:
stmts []Stmt // all the statements in the source file stmts []Stmt // all the statements in the source file
imports []Import // all the imports imports []Import // all the imports
auto_imports []string // imports that were implicitely added auto_imports []string // imports that were implicitely added
embedded_files []EmbeddedFile // list of files to embed in the binary
imported_symbols map[string]string // used for `import {symbol}`, it maps symbol => module.symbol imported_symbols map[string]string // used for `import {symbol}`, it maps symbol => module.symbol
errors []errors.Error // all the checker errors in the file errors []errors.Error // all the checker errors in the file
warnings []errors.Warning // all the checker warings in the file warnings []errors.Warning // all the checker warings in the file
@ -1088,6 +1095,8 @@ pub:
is_vweb bool is_vweb bool
vweb_tmpl File vweb_tmpl File
args_var string args_var string
is_embed bool
embed_file EmbeddedFile
pub mut: pub mut:
sym table.TypeSymbol sym table.TypeSymbol
} }

View File

@ -3248,6 +3248,10 @@ pub fn (mut c Checker) expr(node ast.Expr) table.Type {
} }
ast.ComptimeCall { ast.ComptimeCall {
node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left))) node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left)))
if node.is_embed {
c.file.embedded_files << node.embed_file
return c.table.find_type_idx('embed.EmbeddedData')
}
if node.is_vweb { if node.is_vweb {
// TODO assoc parser bug // TODO assoc parser bug
pref := *c.pref pref := *c.pref

View File

@ -969,6 +969,9 @@ pub fn (mut f Fmt) expr(node ast.Expr) {
} else { } else {
f.write("\$tmpl('$node.args_var')") f.write("\$tmpl('$node.args_var')")
} }
} else {
if node.is_embed {
f.write("\$embed_file('$node.embed_file.rpath')")
} else { } else {
method_expr := if node.has_parens { method_expr := if node.has_parens {
'(${node.method_name}($node.args_var))' '(${node.method_name}($node.args_var))'
@ -978,6 +981,7 @@ pub fn (mut f Fmt) expr(node ast.Expr) {
f.write('${node.left}.$$method_expr') f.write('${node.left}.$$method_expr')
} }
} }
}
ast.ComptimeSelector { ast.ComptimeSelector {
field_expr := if node.has_parens { '($node.field_expr)' } else { node.field_expr.str() } field_expr := if node.has_parens { '($node.field_expr)' } else { node.field_expr.str() }
f.write('${node.left}.$$field_expr') f.write('${node.left}.$$field_expr')

View File

@ -0,0 +1,8 @@
fn main() {
mut the_png := $embed_file('v.png')
println(the_png)
content := the_png.data()
eprintln('content: ${ptr_str(content)}')
eprintln(unsafe { the_png.data().vbytes(the_png.len) }.hex())
println(the_png)
}

View File

@ -44,6 +44,7 @@ mut:
comptime_defines strings.Builder // custom defines, given by -d/-define flags on the CLI comptime_defines strings.Builder // custom defines, given by -d/-define flags on the CLI
pcs_declarations strings.Builder // -prof profile counter declarations for each function pcs_declarations strings.Builder // -prof profile counter declarations for each function
hotcode_definitions strings.Builder // -live declarations & functions hotcode_definitions strings.Builder // -live declarations & functions
embedded_data strings.Builder // data to embed in the executable/binary
shared_types strings.Builder // shared/lock types shared_types strings.Builder // shared/lock types
channel_definitions strings.Builder // channel related code channel_definitions strings.Builder // channel related code
options_typedefs strings.Builder // Option typedefs options_typedefs strings.Builder // Option typedefs
@ -97,6 +98,7 @@ mut:
pcs []ProfileCounterMeta // -prof profile counter fn_names => fn counter name pcs []ProfileCounterMeta // -prof profile counter fn_names => fn counter name
is_builtin_mod bool is_builtin_mod bool
hotcode_fn_names []string hotcode_fn_names []string
embedded_files []ast.EmbeddedFile
// cur_fn ast.FnDecl // cur_fn ast.FnDecl
cur_generic_type table.Type // `int`, `string`, etc in `foo<T>()` cur_generic_type table.Type // `int`, `string`, etc in `foo<T>()`
sql_i int sql_i int
@ -176,6 +178,7 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string
comptime_defines: strings.new_builder(100) comptime_defines: strings.new_builder(100)
pcs_declarations: strings.new_builder(100) pcs_declarations: strings.new_builder(100)
hotcode_definitions: strings.new_builder(100) hotcode_definitions: strings.new_builder(100)
embedded_data: strings.new_builder(1000)
options_typedefs: strings.new_builder(100) options_typedefs: strings.new_builder(100)
options: strings.new_builder(100) options: strings.new_builder(100)
shared_types: strings.new_builder(100) shared_types: strings.new_builder(100)
@ -228,6 +231,14 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string
tests_inited = true tests_inited = true
} }
g.stmts(file.stmts) g.stmts(file.stmts)
// Transfer embedded files
if file.embedded_files.len > 0 {
for path in file.embedded_files {
if path !in g.embedded_files {
g.embedded_files << path
}
}
}
g.timers.show('cgen_file $file.path') g.timers.show('cgen_file $file.path')
} }
g.timers.start('cgen common') g.timers.start('cgen common')
@ -299,6 +310,10 @@ pub fn cgen(files []ast.File, table &table.Table, pref &pref.Preferences) string
b.writeln('\n// V hotcode definitions:') b.writeln('\n// V hotcode definitions:')
b.write(g.hotcode_definitions.str()) b.write(g.hotcode_definitions.str())
} }
if g.embedded_data.len > 0 {
b.writeln('\n// V embedded data:')
b.write(g.embedded_data.str())
}
if g.options_typedefs.len > 0 { if g.options_typedefs.len > 0 {
b.writeln('\n// V option typedefs:') b.writeln('\n// V option typedefs:')
b.write(g.options_typedefs.str()) b.write(g.options_typedefs.str())
@ -421,6 +436,9 @@ pub fn (mut g Gen) finish() {
if g.pref.is_livemain || g.pref.is_liveshared { if g.pref.is_livemain || g.pref.is_liveshared {
g.generate_hotcode_reloader_code() g.generate_hotcode_reloader_code()
} }
if g.pref.is_prod && g.embedded_files.len > 0 {
g.gen_embedded_data()
}
if g.pref.is_test { if g.pref.is_test {
g.gen_c_main_for_tests() g.gen_c_main_for_tests()
} else { } else {

View File

@ -28,6 +28,10 @@ fn (mut g Gen) comptime_selector(node ast.ComptimeSelector) {
} }
fn (mut g Gen) comptime_call(node ast.ComptimeCall) { fn (mut g Gen) comptime_call(node ast.ComptimeCall) {
if node.is_embed {
g.gen_embed_file_init(node)
return
}
if node.is_vweb { if node.is_vweb {
is_html := node.method_name == 'html' is_html := node.method_name == 'html'
for stmt in node.vweb_tmpl.stmts { for stmt in node.vweb_tmpl.stmts {

75
vlib/v/gen/embed.v 100644
View File

@ -0,0 +1,75 @@
module gen
import os
import v.ast
// gen_embed_file_struct generates C code for `$embed_file('...')` calls.
fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) {
g.writeln('(embed__EmbeddedData){')
g.writeln('\t.path = ${ctoslit(node.embed_file.rpath)},')
g.writeln('\t.apath = ${ctoslit(node.embed_file.apath)},')
file_size := os.file_size(node.embed_file.apath)
if file_size > 5242880 {
eprintln('Warning: embedding of files >= ~5MB is currently not supported')
}
if g.pref.is_prod {
// Use function generated in Gen.gen_embedded_data()
g.writeln('\t.compressed = _v_embed_locate_data(${ctoslit(node.embed_file.apath)}),')
}
g.writeln('\t.len = $file_size')
g.writeln('} // $' + 'embed_file("$node.embed_file.apath")')
}
// gen_embedded_data embeds data into the V target executable.
fn (mut g Gen) gen_embedded_data() {
/*
TODO implement compression.
See also the vlib/embed module where decompression should occur.
*/
/*
TODO implement support for large files - right now the setup has problems
// with even just 10 - 50 MB files - the problem is both in V and C compilers.
// maybe we need to write to separate files or have an external tool for large files
// like the `rcc` tool in Qt?
*/
for i, emfile in g.embedded_files {
fbytes := os.read_bytes(emfile.apath) or { panic('Error while embedding file: $err') }
g.embedded_data.write('static const unsigned char _v_embed_blob_$i[$fbytes.len] = {\n ')
for j := 0; j < fbytes.len; j++ {
b := fbytes[j].hex()
if j < fbytes.len - 1 {
g.embedded_data.write('0x$b,')
} else {
g.embedded_data.write('0x$b')
}
if 0 == ((j + 1) % 16) {
g.embedded_data.write('\n ')
}
}
g.embedded_data.writeln('\n};')
}
g.embedded_data.writeln('')
g.embedded_data.writeln('const struct _v_embed {')
g.embedded_data.writeln('\tstring id;')
g.embedded_data.writeln('\tbyteptr data;')
g.embedded_data.writeln('}')
g.embedded_data.writeln('_v_embedded_data[] = {')
for i, emfile in g.embedded_files {
g.embedded_data.writeln('\t{${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
}
g.embedded_data.writeln('\t{_SLIT(""), NULL}')
g.embedded_data.writeln('};')
// See `vlib/v/gen/comptime.v` -> Gen.comptime_call_embed_file(), where this is called at runtime.
// Generate function to locate the data.
g.embedded_data.writeln('
// function to locate embedded data by a vstring
byteptr _v_embed_locate_data(string id) {
const struct _v_embed *ve;
for (ve = _v_embedded_data; !string_eq(ve->id, _SLIT("")) && ve->data != NULL; ve++) {
if (string_eq(ve->id, id)) {
return (byteptr) ve->data;
}
}
return NULL;
}')
}

View File

@ -10,6 +10,10 @@ import v.table
import v.token import v.token
import vweb.tmpl import vweb.tmpl
const (
supported_comptime_calls = ['html', 'tmpl', 'embed_file']
)
// // #include, #flag, #v // // #include, #flag, #v
fn (mut p Parser) hash() ast.HashStmt { fn (mut p Parser) hash() ast.HashStmt {
mut pos := p.prev_tok.position() mut pos := p.prev_tok.position()
@ -37,9 +41,9 @@ fn (mut p Parser) hash() ast.HashStmt {
} }
} }
fn (mut p Parser) vweb() ast.ComptimeCall { fn (mut p Parser) comp_call() ast.ComptimeCall {
p.check(.dollar) p.check(.dollar)
error_msg := 'only `\$tmpl()` and `\$vweb.html()` comptime functions are supported right now' error_msg := 'only `\$tmpl()`, `\$embed_file()` and `\$vweb.html()` comptime functions are supported right now'
if p.peek_tok.kind == .dot { if p.peek_tok.kind == .dot {
n := p.check_name() // skip `vweb.html()` TODO n := p.check_name() // skip `vweb.html()` TODO
if n != 'vweb' { if n != 'vweb' {
@ -49,17 +53,57 @@ fn (mut p Parser) vweb() ast.ComptimeCall {
p.check(.dot) p.check(.dot)
} }
n := p.check_name() // (.name) n := p.check_name() // (.name)
if n != 'html' && n != 'tmpl' { if n !in supported_comptime_calls {
p.error(error_msg) p.error(error_msg)
return ast.ComptimeCall{} return ast.ComptimeCall{}
} }
is_embed_file := n == 'embed_file'
is_html := n == 'html' is_html := n == 'html'
p.check(.lpar) p.check(.lpar)
spos := p.tok.position()
s := if is_html { '' } else { p.tok.lit } s := if is_html { '' } else { p.tok.lit }
if !is_html { if !is_html {
p.check(.string) p.check(.string)
} }
p.check(.rpar) p.check(.rpar)
//
if is_embed_file {
mut epath := s
// Validate that the epath exists, and that it is actually a file.
if epath == '' {
p.error_with_pos('please supply a valid relative or absolute file path to the file to embed',
spos)
return ast.ComptimeCall{}
}
if !p.pref.is_fmt {
abs_path := os.real_path(epath)
// check absolute path first
if !os.exists(abs_path) {
// ... look relative to the source file:
epath = os.real_path(os.join_path(os.dir(p.file_name), epath))
if !os.exists(epath) {
p.error_with_pos('"$epath" does not exist so it cannot be embedded',
spos)
return ast.ComptimeCall{}
}
if !os.is_file(epath) {
p.error_with_pos('"$epath" is not a file so it cannot be embedded',
spos)
return ast.ComptimeCall{}
}
} else {
epath = abs_path
}
}
p.register_auto_import('embed')
return ast.ComptimeCall{
is_embed: true
embed_file: ast.EmbeddedFile{
rpath: s
apath: epath
}
}
}
// Compile vweb html template to V code, parse that V code and embed the resulting V function // Compile vweb html template to V code, parse that V code and embed the resulting V function
// that returns an html string. // that returns an html string.
fn_path := p.cur_fn_name.split('_') fn_path := p.cur_fn_name.split('_')
@ -71,6 +115,7 @@ fn (mut p Parser) vweb() ast.ComptimeCall {
if !is_html { if !is_html {
path = tmpl_path path = tmpl_path
} }
eprintln('>>> is_embed_file: $is_embed_file | is_html: $is_html | s: $s | n: $n | path: $path')
if !os.exists(path) { if !os.exists(path) {
// can be in `templates/` // can be in `templates/`
if is_html { if is_html {

View File

@ -696,7 +696,7 @@ pub fn (mut p Parser) stmt(is_top_level bool) ast.Stmt {
} }
.name { .name {
return ast.ExprStmt{ return ast.ExprStmt{
expr: p.vweb() expr: p.comp_call()
} }
} }
else { else {

View File

@ -56,7 +56,7 @@ pub fn (mut p Parser) expr(precedence int) ast.Expr {
.dollar { .dollar {
match p.peek_tok.kind { match p.peek_tok.kind {
.name { .name {
return p.vweb() return p.comp_call()
} }
.key_if { .key_if {
return p.if_expr(true) return p.if_expr(true)

View File

@ -0,0 +1,25 @@
const const_file = $embed_file('v.png')
fn test_const_embed_file() {
mut file := const_file
eprintln('file: $file')
assert file.len == 603
fdata := file.data()
eprintln('file after .data() call: $file')
assert file.len == 603
unsafe {
assert fdata.vbytes(4) == [byte(0x89), `P`, `N`, `G`]
}
}
fn test_embed_file() {
mut file := $embed_file('v.png')
eprintln('file: $file')
assert file.len == 603
fdata := file.data()
eprintln('file after .data() call: $file')
assert file.len == 603
unsafe {
assert fdata.vbytes(4) == [byte(0x89), `P`, `N`, `G`]
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 603 B