vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

embed: move to `v.embed_file`, fix CI failing test

pull/8150/head^2
Delyan Angelov 2021-01-16 20:03:07 +02:00
parent d258733752
commit 5ae55731b9
No known key found for this signature in database
GPG Key ID: 66886C0F12D595ED
12 changed files with 146 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
vlib/embed/embed.v
View File

@ -1,61 +0,0 @@
module embed
import os
// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
// EmbeddedData encapsulates functionality for the `$embed_file()` compile time call.
pub struct EmbeddedData {
path string
apath string
mut:
compressed byteptr
uncompressed byteptr
free_compressed bool
free_uncompressed bool
pub:
len int
}
pub fn (ed EmbeddedData) str() string {
return 'embed.EmbeddedData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
}
[unsafe]
pub fn (mut ed EmbeddedData) free() {
unsafe {
ed.path.free()
ed.apath.free()
if ed.free_compressed {
free(ed.compressed)
}
if ed.free_uncompressed {
free(ed.uncompressed)
}
}
}
pub fn (mut ed EmbeddedData) data() byteptr {
if !isnil(ed.uncompressed) {
return ed.uncompressed
} else {
if isnil(ed.uncompressed) && !isnil(ed.compressed) {
// TODO implement uncompression
// See also C Gen.gen_embedded_data() where the compression should occur.
ed.uncompressed = ed.compressed
} else {
mut path := os.resource_abs_path(ed.path)
if !os.is_file(path) {
path = ed.apath
if !os.is_file(path) {
panic('EmbeddedData error: files "$ed.path" and "$ed.apath" do not exist')
}
}
bytes := os.read_bytes(path) or {
panic('EmbeddedData error: "$path" could not be read: $err')
}
ed.uncompressed = bytes.data
ed.free_uncompressed = true
}
}
return ed.uncompressed
}

2
vlib/v/checker/checker.v
View File

@ -3295,7 +3295,7 @@ pub fn (mut c Checker) expr(node ast.Expr) table.Type {
node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left)))
if node.is_embed {
c.file.embedded_files << node.embed_file
return c.table.find_type_idx('embed.EmbeddedData')
return c.table.find_type_idx('v.embed_file.EmbedFileData')
}
if node.is_vweb {
// TODO assoc parser bug

89
vlib/v/embed_file/embed_file.v 100644
View File

@ -0,0 +1,89 @@
module embed_file
import os
// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
// EmbedFileData encapsulates functionality for the `$embed_file()` compile time call.
pub struct EmbedFileData {
path string
apath string
mut:
compressed byteptr
uncompressed byteptr
free_compressed bool
free_uncompressed bool
pub:
len int
}
pub fn (ed EmbedFileData) str() string {
return 'embed_file.EmbedFileData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
}
[unsafe]
pub fn (mut ed EmbedFileData) free() {
unsafe {
ed.path.free()
ed.apath.free()
if ed.free_compressed {
free(ed.compressed)
ed.compressed = byteptr(0)
}
if ed.free_uncompressed {
free(ed.uncompressed)
ed.uncompressed = byteptr(0)
}
}
}
pub fn (mut ed EmbedFileData) data() byteptr {
if !isnil(ed.uncompressed) {
return ed.uncompressed
} else {
if isnil(ed.uncompressed) && !isnil(ed.compressed) {
// TODO implement uncompression
// See also C Gen.gen_embedded_data() where the compression should occur.
ed.uncompressed = ed.compressed
} else {
mut path := os.resource_abs_path(ed.path)
if !os.is_file(path) {
path = ed.apath
if !os.is_file(path) {
panic('EmbedFileData error: files "$ed.path" and "$ed.apath" do not exist')
}
}
bytes := os.read_bytes(path) or {
panic('EmbedFileData error: "$path" could not be read: $err')
}
ed.uncompressed = bytes.data
ed.free_uncompressed = true
}
}
return ed.uncompressed
}
//////////////////////////////////////////////////////////////////////////////
// EmbedFileIndexEntry is used internally by the V compiler when you compile a
// program that uses $embed_file('file.bin') in -prod mode.
// V will generate a static index of all embedded files, and will call the
// find_index_entry_by_path over the index and the relative paths of the embeds.
// NB: these are public on purpose, to help -usecache.
pub struct EmbedFileIndexEntry {
id int
path string
data byteptr
}
// find_index_entry_by_path is used internally by the V compiler:
pub fn find_index_entry_by_path(start voidptr, path string) &EmbedFileIndexEntry {
mut x := &EmbedFileIndexEntry(start)
for !(x.path == path || isnil(x.data)) {
unsafe {
x = &EmbedFileIndexEntry(u64(x) + sizeof(EmbedFileIndexEntry))
}
}
$if debug_embed_file_in_prod ? {
eprintln('>> v.embed_file find_index_entry_by_path ${ptr_str(start)}, path: "$path" => ${ptr_str(x)}')
}
return x
}

0
vlib/v/tests/embed_file/embed_file_test.v → vlib/v/embed_file/embed_file_test.v
View File

0
vlib/v/tests/embed_file/v.png → vlib/v/embed_file/v.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 603 B

After

Width:  |  Height:  |  Size: 603 B

2
vlib/v/gen/cgen.v
View File

@ -443,7 +443,7 @@ pub fn (mut g Gen) finish() {
if g.pref.is_livemain || g.pref.is_liveshared {
g.generate_hotcode_reloader_code()
}
if g.pref.is_prod && g.embedded_files.len > 0 {
if g.embed_file_is_prod_mode() && g.embedded_files.len > 0 {
g.gen_embedded_data()
}
if g.pref.is_test {

46
vlib/v/gen/embed.v
View File

@ -3,20 +3,30 @@ module gen
import os
import v.ast
fn (mut g Gen) embed_file_is_prod_mode() bool {
if g.pref.is_prod || 'debug_embed_file_in_prod' in g.pref.compile_defines {
return true
}
return false
}
// gen_embed_file_struct generates C code for `$embed_file('...')` calls.
fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) {
g.writeln('(embed__EmbeddedData){')
g.writeln('\t.path = ${ctoslit(node.embed_file.rpath)},')
g.writeln('\t.apath = ${ctoslit(node.embed_file.apath)},')
g.writeln('(v__embed_file__EmbedFileData){')
g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},')
g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},')
file_size := os.file_size(node.embed_file.apath)
if file_size > 5242880 {
eprintln('Warning: embedding of files >= ~5MB is currently not supported')
}
if g.pref.is_prod {
if g.embed_file_is_prod_mode() {
// Use function generated in Gen.gen_embedded_data()
g.writeln('\t.compressed = _v_embed_locate_data(${ctoslit(node.embed_file.apath)}),')
g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)})->data,')
}
g.writeln('\t.len = $file_size')
g.writeln('\t\t.uncompressed = NULL,')
g.writeln('\t\t.free_compressed = 0,')
g.writeln('\t\t.free_uncompressed = 0,')
g.writeln('\t\t.len = $file_size')
g.writeln('} // $' + 'embed_file("$node.embed_file.apath")')
}
@ -49,27 +59,11 @@ fn (mut g Gen) gen_embedded_data() {
g.embedded_data.writeln('\n};')
}
g.embedded_data.writeln('')
g.embedded_data.writeln('const struct _v_embed {')
g.embedded_data.writeln('\tstring id;')
g.embedded_data.writeln('\tbyteptr data;')
g.embedded_data.writeln('}')
g.embedded_data.writeln('_v_embedded_data[] = {')
g.embedded_data.writeln('const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[] = {')
for i, emfile in g.embedded_files {
g.embedded_data.writeln('\t{${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
g.embedded_data.writeln('\t{$i, ${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
}
g.embedded_data.writeln('\t{_SLIT(""), NULL}')
g.embedded_data.writeln('\t{-1, _SLIT(""), NULL}')
g.embedded_data.writeln('};')
// See `vlib/v/gen/comptime.v` -> Gen.comptime_call_embed_file(), where this is called at runtime.
// Generate function to locate the data.
g.embedded_data.writeln('
// function to locate embedded data by a vstring
byteptr _v_embed_locate_data(string id) {
const struct _v_embed *ve;
for (ve = _v_embedded_data; !string_eq(ve->id, _SLIT("")) && ve->data != NULL; ve++) {
if (string_eq(ve->id, id)) {
return (byteptr) ve->data;
}
}
return NULL;
}')
// see vlib/v/embed_file/embed_file.v, find_index_entry_by_id/2 and find_index_entry_by_path/2
}

65
vlib/v/live/executable/reloader.v
View File

@ -11,7 +11,6 @@ pub const (
)
// The live reloader code is implemented here.
// NB: new_live_reload_info will be called by generated C code inside main()
pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_mutex voidptr, live_linkfn live.FNLinkLiveSymbols) &live.LiveReloadInfo {
file_base := os.file_name(original).replace('.v', '')
@ -28,7 +27,7 @@ pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_
live_fn_mutex: live_fn_mutex
live_linkfn: live_linkfn
so_extension: so_extension
so_name_template: '${so_dir}/tmp.%d.${file_base}'
so_name_template: '$so_dir/tmp.%d.$file_base'
live_lib: 0
reloads: 0
reload_time_ms: 0
@ -42,15 +41,15 @@ pub fn start_reloader(mut r live.LiveReloadInfo) {
// The shared library should be loaded once in the main thread
// If that fails, the program would crash anyway, just provide
// an error message to the user and exit:
r.reloads++
r.reloads++
compile_and_reload_shared_lib(mut r) or {
eprintln( err )
eprintln(err)
exit(1)
}
go reloader(mut r)
}
fn elog(r &live.LiveReloadInfo, s string){
fn elog(r &live.LiveReloadInfo, s string) {
$if debuglive ? {
eprintln(s)
}
@ -58,11 +57,9 @@ fn elog(r &live.LiveReloadInfo, s string){
fn compile_and_reload_shared_lib(mut r live.LiveReloadInfo) ?bool {
sw := time.new_stopwatch({})
new_lib_path := compile_lib(mut r) or {
return error('errors while compiling $r.original')
}
elog(r,'> compile_and_reload_shared_lib compiled: ${new_lib_path}')
load_lib(mut r, new_lib_path )
new_lib_path := compile_lib(mut r) or { return error('errors while compiling $r.original') }
elog(r, '> compile_and_reload_shared_lib compiled: $new_lib_path')
load_lib(mut r, new_lib_path)
r.reload_time_ms = int(sw.elapsed().milliseconds())
return true
}
@ -70,19 +67,19 @@ fn compile_and_reload_shared_lib(mut r live.LiveReloadInfo) ?bool {
fn compile_lib(mut r live.LiveReloadInfo) ?string {
new_lib_path, new_lib_path_with_extension := current_shared_library_path(mut r)
cmd := '$r.vexe $r.vopts -o $new_lib_path $r.original'
elog(r,'> compilation cmd: $cmd')
elog(r, '> compilation cmd: $cmd')
cwatch := time.new_stopwatch({})
recompilation_result := os.exec( cmd ) or {
recompilation_result := os.exec(cmd) or {
eprintln('recompilation failed')
return none
}
elog(r,'compilation took: ${cwatch.elapsed().milliseconds()}ms')
elog(r, 'compilation took: ${cwatch.elapsed().milliseconds()}ms')
if recompilation_result.exit_code != 0 {
eprintln('recompilation error:')
eprintln( recompilation_result.output )
eprintln(recompilation_result.output)
return none
}
if !os.exists( new_lib_path_with_extension ) {
if !os.exists(new_lib_path_with_extension) {
eprintln('new_lib_path: $new_lib_path_with_extension does not exist')
return none
}
@ -96,29 +93,29 @@ fn current_shared_library_path(mut r live.LiveReloadInfo) (string, string) {
}
fn load_lib(mut r live.LiveReloadInfo, new_lib_path string) {
elog(r,'live mutex locking...')
elog(r, 'live mutex locking...')
C.pthread_mutex_lock(r.live_fn_mutex)
elog(r,'live mutex locked')
elog(r, 'live mutex locked')
//
if r.cb_locked_before != voidptr(0) {
r.cb_locked_before( r )
r.cb_locked_before(r)
}
//
protected_load_lib(mut r, new_lib_path)
//
r.reloads_ok++
if r.cb_locked_after != voidptr(0) {
r.cb_locked_after( r )
r.cb_locked_after(r)
}
//
elog(r,'live mutex unlocking...')
elog(r, 'live mutex unlocking...')
C.pthread_mutex_unlock(r.live_fn_mutex)
elog(r,'live mutex unlocked')
elog(r, 'live mutex unlocked')
}
fn protected_load_lib(mut r live.LiveReloadInfo, new_lib_path string) {
if r.live_lib != 0 {
dl.close( r.live_lib )
dl.close(r.live_lib)
r.live_lib = C.NULL
}
r.live_lib = dl.open(new_lib_path, dl.rtld_lazy)
@ -126,40 +123,40 @@ fn protected_load_lib(mut r live.LiveReloadInfo, new_lib_path string) {
eprintln('opening $new_lib_path failed')
exit(1)
}
r.live_linkfn( r.live_lib )
elog(r,'> load_lib OK, new live_lib: $r.live_lib')
r.live_linkfn(r.live_lib)
elog(r, '> load_lib OK, new live_lib: $r.live_lib')
// removing the .so file from the filesystem after dlopen-ing
// it is safe, since it will still be mapped in memory
os.rm( new_lib_path )
// it is safe, since it will still be mapped in memory
os.rm(new_lib_path)
}
// NB: r.reloader() is executed in a new, independent thread
fn reloader(mut r live.LiveReloadInfo) {
// elog(r,'reloader, r: $r')
mut last_ts := os.file_last_mod_unix( r.original )
// elog(r,'reloader, r: $r')
mut last_ts := os.file_last_mod_unix(r.original)
for {
if r.cb_recheck != voidptr(0) {
r.cb_recheck( r )
r.cb_recheck(r)
}
now_ts := os.file_last_mod_unix( r.original )
now_ts := os.file_last_mod_unix(r.original)
if last_ts != now_ts {
r.reloads++
last_ts = now_ts
r.last_mod_ts = last_ts
if r.cb_before != voidptr(0) {
r.cb_before( r )
r.cb_before(r)
}
compile_and_reload_shared_lib(mut r) or {
if r.cb_compile_failed != voidptr(0) {
r.cb_compile_failed( r )
r.cb_compile_failed(r)
}
if r.cb_after != voidptr(0) {
r.cb_after( r )
r.cb_after(r)
}
continue
}
if r.cb_after != voidptr(0) {
r.cb_after( r )
r.cb_after(r)
}
}
if r.recheck_period_ms > 0 {

4
vlib/v/live/live_test.v
View File

@ -44,9 +44,7 @@ const (
)
fn get_source_template() string {
src := os.read_file(os.join_path(os.dir(@FILE), 'live_test_template.vv')) or {
panic(err)
}
src := os.read_file(os.join_path(os.dir(@FILE), 'live_test_template.vv')) or { panic(err) }
return src.replace('#OUTPUT_FILE#', output_file)
}

2
vlib/v/parser/comptime.v
View File

@ -95,7 +95,7 @@ fn (mut p Parser) comp_call() ast.ComptimeCall {
epath = abs_path
}
}
p.register_auto_import('embed')
p.register_auto_import('v.embed_file')
return ast.ComptimeCall{
is_embed: true
embed_file: ast.EmbeddedFile{

2
vlib/v/pkgconfig/pkgconfig.v
View File

@ -43,7 +43,7 @@ pub mut:
}
fn (mut pc PkgConfig) parse_list_no_comma(s string) []string {
return pc.parse_list( s.replace(',', ' ') )
return pc.parse_list(s.replace(',', ' '))
}
fn (mut pc PkgConfig) parse_list(s string) []string {

2
vlib/v/pref/pref.v
View File

@ -114,7 +114,7 @@ pub mut:
display_name string
bundle_id string
path string // Path to file/folder to compile
// -d vfmt and -d another=0 for `$if vfmt { will execute }` and `$if another { will NOT get here }`
// -d vfmt and -d another=0 for `$if vfmt { will execute }` and `$if another ? { will NOT get here }`
compile_defines []string // just ['vfmt']
compile_defines_all []string // contains both: ['vfmt','another']
run_args []string // `v run x.v 1 2 3` => `1 2 3`