embed: move to `v.embed_file`, fix CI failing test

pull/8150/head^2
Delyan Angelov 2021-01-16 20:03:07 +02:00
parent d258733752
commit 5ae55731b9
No known key found for this signature in database
GPG Key ID: 66886C0F12D595ED
12 changed files with 146 additions and 129 deletions

View File

@ -1,61 +0,0 @@
module embed
import os
// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
// EmbeddedData encapsulates functionality for the `$embed_file()` compile time call.
pub struct EmbeddedData {
path string
apath string
mut:
compressed byteptr
uncompressed byteptr
free_compressed bool
free_uncompressed bool
pub:
len int
}
pub fn (ed EmbeddedData) str() string {
return 'embed.EmbeddedData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
}
[unsafe]
pub fn (mut ed EmbeddedData) free() {
unsafe {
ed.path.free()
ed.apath.free()
if ed.free_compressed {
free(ed.compressed)
}
if ed.free_uncompressed {
free(ed.uncompressed)
}
}
}
pub fn (mut ed EmbeddedData) data() byteptr {
if !isnil(ed.uncompressed) {
return ed.uncompressed
} else {
if isnil(ed.uncompressed) && !isnil(ed.compressed) {
// TODO implement uncompression
// See also C Gen.gen_embedded_data() where the compression should occur.
ed.uncompressed = ed.compressed
} else {
mut path := os.resource_abs_path(ed.path)
if !os.is_file(path) {
path = ed.apath
if !os.is_file(path) {
panic('EmbeddedData error: files "$ed.path" and "$ed.apath" do not exist')
}
}
bytes := os.read_bytes(path) or {
panic('EmbeddedData error: "$path" could not be read: $err')
}
ed.uncompressed = bytes.data
ed.free_uncompressed = true
}
}
return ed.uncompressed
}

View File

@ -3295,7 +3295,7 @@ pub fn (mut c Checker) expr(node ast.Expr) table.Type {
node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left)))
if node.is_embed {
c.file.embedded_files << node.embed_file
return c.table.find_type_idx('embed.EmbeddedData')
return c.table.find_type_idx('v.embed_file.EmbedFileData')
}
if node.is_vweb {
// TODO assoc parser bug

View File

@ -0,0 +1,89 @@
module embed_file
import os
// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
// EmbedFileData encapsulates functionality for the `$embed_file()` compile time call.
pub struct EmbedFileData {
path string
apath string
mut:
compressed byteptr
uncompressed byteptr
free_compressed bool
free_uncompressed bool
pub:
len int
}
pub fn (ed EmbedFileData) str() string {
return 'embed_file.EmbedFileData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
}
[unsafe]
pub fn (mut ed EmbedFileData) free() {
unsafe {
ed.path.free()
ed.apath.free()
if ed.free_compressed {
free(ed.compressed)
ed.compressed = byteptr(0)
}
if ed.free_uncompressed {
free(ed.uncompressed)
ed.uncompressed = byteptr(0)
}
}
}
pub fn (mut ed EmbedFileData) data() byteptr {
if !isnil(ed.uncompressed) {
return ed.uncompressed
} else {
if isnil(ed.uncompressed) && !isnil(ed.compressed) {
// TODO implement uncompression
// See also C Gen.gen_embedded_data() where the compression should occur.
ed.uncompressed = ed.compressed
} else {
mut path := os.resource_abs_path(ed.path)
if !os.is_file(path) {
path = ed.apath
if !os.is_file(path) {
panic('EmbedFileData error: files "$ed.path" and "$ed.apath" do not exist')
}
}
bytes := os.read_bytes(path) or {
panic('EmbedFileData error: "$path" could not be read: $err')
}
ed.uncompressed = bytes.data
ed.free_uncompressed = true
}
}
return ed.uncompressed
}
//////////////////////////////////////////////////////////////////////////////
// EmbedFileIndexEntry is used internally by the V compiler when you compile a
// program that uses $embed_file('file.bin') in -prod mode.
// V will generate a static index of all embedded files, and will call the
// find_index_entry_by_path over the index and the relative paths of the embeds.
// NB: these are public on purpose, to help -usecache.
pub struct EmbedFileIndexEntry {
id int
path string
data byteptr
}
// find_index_entry_by_path is used internally by the V compiler:
pub fn find_index_entry_by_path(start voidptr, path string) &EmbedFileIndexEntry {
mut x := &EmbedFileIndexEntry(start)
for !(x.path == path || isnil(x.data)) {
unsafe {
x = &EmbedFileIndexEntry(u64(x) + sizeof(EmbedFileIndexEntry))
}
}
$if debug_embed_file_in_prod ? {
eprintln('>> v.embed_file find_index_entry_by_path ${ptr_str(start)}, path: "$path" => ${ptr_str(x)}')
}
return x
}

View File

Before

Width:  |  Height:  |  Size: 603 B

After

Width:  |  Height:  |  Size: 603 B

View File

@ -443,7 +443,7 @@ pub fn (mut g Gen) finish() {
if g.pref.is_livemain || g.pref.is_liveshared {
g.generate_hotcode_reloader_code()
}
if g.pref.is_prod && g.embedded_files.len > 0 {
if g.embed_file_is_prod_mode() && g.embedded_files.len > 0 {
g.gen_embedded_data()
}
if g.pref.is_test {

View File

@ -3,20 +3,30 @@ module gen
import os
import v.ast
fn (mut g Gen) embed_file_is_prod_mode() bool {
if g.pref.is_prod || 'debug_embed_file_in_prod' in g.pref.compile_defines {
return true
}
return false
}
// gen_embed_file_struct generates C code for `$embed_file('...')` calls.
fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) {
g.writeln('(embed__EmbeddedData){')
g.writeln('\t.path = ${ctoslit(node.embed_file.rpath)},')
g.writeln('\t.apath = ${ctoslit(node.embed_file.apath)},')
g.writeln('(v__embed_file__EmbedFileData){')
g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},')
g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},')
file_size := os.file_size(node.embed_file.apath)
if file_size > 5242880 {
eprintln('Warning: embedding of files >= ~5MB is currently not supported')
}
if g.pref.is_prod {
if g.embed_file_is_prod_mode() {
// Use function generated in Gen.gen_embedded_data()
g.writeln('\t.compressed = _v_embed_locate_data(${ctoslit(node.embed_file.apath)}),')
g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)})->data,')
}
g.writeln('\t.len = $file_size')
g.writeln('\t\t.uncompressed = NULL,')
g.writeln('\t\t.free_compressed = 0,')
g.writeln('\t\t.free_uncompressed = 0,')
g.writeln('\t\t.len = $file_size')
g.writeln('} // $' + 'embed_file("$node.embed_file.apath")')
}
@ -49,27 +59,11 @@ fn (mut g Gen) gen_embedded_data() {
g.embedded_data.writeln('\n};')
}
g.embedded_data.writeln('')
g.embedded_data.writeln('const struct _v_embed {')
g.embedded_data.writeln('\tstring id;')
g.embedded_data.writeln('\tbyteptr data;')
g.embedded_data.writeln('}')
g.embedded_data.writeln('_v_embedded_data[] = {')
g.embedded_data.writeln('const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[] = {')
for i, emfile in g.embedded_files {
g.embedded_data.writeln('\t{${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
g.embedded_data.writeln('\t{$i, ${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
}
g.embedded_data.writeln('\t{_SLIT(""), NULL}')
g.embedded_data.writeln('\t{-1, _SLIT(""), NULL}')
g.embedded_data.writeln('};')
// See `vlib/v/gen/comptime.v` -> Gen.comptime_call_embed_file(), where this is called at runtime.
// Generate function to locate the data.
g.embedded_data.writeln('
// function to locate embedded data by a vstring
byteptr _v_embed_locate_data(string id) {
const struct _v_embed *ve;
for (ve = _v_embedded_data; !string_eq(ve->id, _SLIT("")) && ve->data != NULL; ve++) {
if (string_eq(ve->id, id)) {
return (byteptr) ve->data;
}
}
return NULL;
}')
// see vlib/v/embed_file/embed_file.v, find_index_entry_by_id/2 and find_index_entry_by_path/2
}

View File

@ -11,7 +11,6 @@ pub const (
)
// The live reloader code is implemented here.
// NB: new_live_reload_info will be called by generated C code inside main()
pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_mutex voidptr, live_linkfn live.FNLinkLiveSymbols) &live.LiveReloadInfo {
file_base := os.file_name(original).replace('.v', '')
@ -28,7 +27,7 @@ pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_
live_fn_mutex: live_fn_mutex
live_linkfn: live_linkfn
so_extension: so_extension
so_name_template: '${so_dir}/tmp.%d.${file_base}'
so_name_template: '$so_dir/tmp.%d.$file_base'
live_lib: 0
reloads: 0
reload_time_ms: 0
@ -58,10 +57,8 @@ fn elog(r &live.LiveReloadInfo, s string){
fn compile_and_reload_shared_lib(mut r live.LiveReloadInfo) ?bool {
sw := time.new_stopwatch({})
new_lib_path := compile_lib(mut r) or {
return error('errors while compiling $r.original')
}
elog(r,'> compile_and_reload_shared_lib compiled: ${new_lib_path}')
new_lib_path := compile_lib(mut r) or { return error('errors while compiling $r.original') }
elog(r, '> compile_and_reload_shared_lib compiled: $new_lib_path')
load_lib(mut r, new_lib_path)
r.reload_time_ms = int(sw.elapsed().milliseconds())
return true

View File

@ -44,9 +44,7 @@ const (
)
fn get_source_template() string {
src := os.read_file(os.join_path(os.dir(@FILE), 'live_test_template.vv')) or {
panic(err)
}
src := os.read_file(os.join_path(os.dir(@FILE), 'live_test_template.vv')) or { panic(err) }
return src.replace('#OUTPUT_FILE#', output_file)
}

View File

@ -95,7 +95,7 @@ fn (mut p Parser) comp_call() ast.ComptimeCall {
epath = abs_path
}
}
p.register_auto_import('embed')
p.register_auto_import('v.embed_file')
return ast.ComptimeCall{
is_embed: true
embed_file: ast.EmbeddedFile{

View File

@ -114,7 +114,7 @@ pub mut:
display_name string
bundle_id string
path string // Path to file/folder to compile
// -d vfmt and -d another=0 for `$if vfmt { will execute }` and `$if another { will NOT get here }`
// -d vfmt and -d another=0 for `$if vfmt { will execute }` and `$if another ? { will NOT get here }`
compile_defines []string // just ['vfmt']
compile_defines_all []string // contains both: ['vfmt','another']
run_args []string // `v run x.v 1 2 3` => `1 2 3`