embed: move to `v.embed_file`, fix CI failing test

vlib/embed/embed.v

@@ -1,61 +0,0 @@
-module embed
-
-import os
-
-// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
-// EmbeddedData encapsulates functionality for the `$embed_file()` compile time call.
-pub struct EmbeddedData {
-	path  string
-	apath string
-mut:
-	compressed        byteptr
-	uncompressed      byteptr
-	free_compressed   bool
-	free_uncompressed bool
-pub:
-	len int
-}
-
-pub fn (ed EmbeddedData) str() string {
-	return 'embed.EmbeddedData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
-}
-
-[unsafe]
-pub fn (mut ed EmbeddedData) free() {
-	unsafe {
-		ed.path.free()
-		ed.apath.free()
-		if ed.free_compressed {
-			free(ed.compressed)
-		}
-		if ed.free_uncompressed {
-			free(ed.uncompressed)
-		}
-	}
-}
-
-pub fn (mut ed EmbeddedData) data() byteptr {
-	if !isnil(ed.uncompressed) {
-		return ed.uncompressed
-	} else {
-		if isnil(ed.uncompressed) && !isnil(ed.compressed) {
-			// TODO implement uncompression
-			// See also C Gen.gen_embedded_data() where the compression should occur.
-			ed.uncompressed = ed.compressed
-		} else {
-			mut path := os.resource_abs_path(ed.path)
-			if !os.is_file(path) {
-				path = ed.apath
-				if !os.is_file(path) {
-					panic('EmbeddedData error: files "$ed.path" and "$ed.apath" do not exist')
-				}
-			}
-			bytes := os.read_bytes(path) or {
-				panic('EmbeddedData error: "$path" could not be read: $err')
-			}
-			ed.uncompressed = bytes.data
-			ed.free_uncompressed = true
-		}
-	}
-	return ed.uncompressed
-}

vlib/v/checker/checker.v

@@ -3295,7 +3295,7 @@ pub fn (mut c Checker) expr(node ast.Expr) table.Type {
 			node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left)))
 			if node.is_embed {
 				c.file.embedded_files << node.embed_file
-				return c.table.find_type_idx('embed.EmbeddedData')
+				return c.table.find_type_idx('v.embed_file.EmbedFileData')
 			}
 			if node.is_vweb {
 				// TODO assoc parser bug

vlib/v/embed_file/embed_file.v

@@ -0,0 +1,89 @@
+module embed_file
+
+import os
+
+// https://github.com/vlang/rfcs/blob/master/embedding_resources.md
+// EmbedFileData encapsulates functionality for the `$embed_file()` compile time call.
+pub struct EmbedFileData {
+	path  string
+	apath string
+mut:
+	compressed        byteptr
+	uncompressed      byteptr
+	free_compressed   bool
+	free_uncompressed bool
+pub:
+	len int
+}
+
+pub fn (ed EmbedFileData) str() string {
+	return 'embed_file.EmbedFileData{ len: $ed.len, path: "$ed.path", path: "$ed.apath", uncompressed: ${ptr_str(ed.uncompressed)} }'
+}
+
+[unsafe]
+pub fn (mut ed EmbedFileData) free() {
+	unsafe {
+		ed.path.free()
+		ed.apath.free()
+		if ed.free_compressed {
+			free(ed.compressed)
+			ed.compressed = byteptr(0)
+		}
+		if ed.free_uncompressed {
+			free(ed.uncompressed)
+			ed.uncompressed = byteptr(0)
+		}
+	}
+}
+
+pub fn (mut ed EmbedFileData) data() byteptr {
+	if !isnil(ed.uncompressed) {
+		return ed.uncompressed
+	} else {
+		if isnil(ed.uncompressed) && !isnil(ed.compressed) {
+			// TODO implement uncompression
+			// See also C Gen.gen_embedded_data() where the compression should occur.
+			ed.uncompressed = ed.compressed
+		} else {
+			mut path := os.resource_abs_path(ed.path)
+			if !os.is_file(path) {
+				path = ed.apath
+				if !os.is_file(path) {
+					panic('EmbedFileData error: files "$ed.path" and "$ed.apath" do not exist')
+				}
+			}
+			bytes := os.read_bytes(path) or {
+				panic('EmbedFileData error: "$path" could not be read: $err')
+			}
+			ed.uncompressed = bytes.data
+			ed.free_uncompressed = true
+		}
+	}
+	return ed.uncompressed
+}
+
+//////////////////////////////////////////////////////////////////////////////
+// EmbedFileIndexEntry is used internally by the V compiler when you compile a
+// program that uses $embed_file('file.bin') in -prod mode.
+// V will generate a static index of all embedded files, and will call the
+// find_index_entry_by_path over the index and the relative paths of the embeds.
+// NB: these are public on purpose, to help -usecache.
+pub struct EmbedFileIndexEntry {
+	id   int
+	path string
+	data byteptr
+}
+
+// find_index_entry_by_path is used internally by the V compiler:
+pub fn find_index_entry_by_path(start voidptr, path string) &EmbedFileIndexEntry {
+	mut x := &EmbedFileIndexEntry(start)
+	for !(x.path == path || isnil(x.data)) {
+		unsafe {
+			x = &EmbedFileIndexEntry(u64(x) + sizeof(EmbedFileIndexEntry))
+		}
+	}
+	$if debug_embed_file_in_prod ? {
+		eprintln('>> v.embed_file find_index_entry_by_path ${ptr_str(start)}, path: "$path" => ${ptr_str(x)}')
+	}
+	return x
+}

vlib/v/gen/cgen.v

@@ -443,7 +443,7 @@ pub fn (mut g Gen) finish() {
 	if g.pref.is_livemain || g.pref.is_liveshared {
 		g.generate_hotcode_reloader_code()
 	}
-	if g.pref.is_prod && g.embedded_files.len > 0 {
+	if g.embed_file_is_prod_mode() && g.embedded_files.len > 0 {
 		g.gen_embedded_data()
 	}
 	if g.pref.is_test {

vlib/v/gen/embed.v

@@ -3,20 +3,30 @@ module gen
 import os
 import v.ast
 
+fn (mut g Gen) embed_file_is_prod_mode() bool {
+	if g.pref.is_prod || 'debug_embed_file_in_prod' in g.pref.compile_defines {
+		return true
+	}
+	return false
+}
+
 // gen_embed_file_struct generates C code for `$embed_file('...')` calls.
 fn (mut g Gen) gen_embed_file_init(node ast.ComptimeCall) {
-	g.writeln('(embed__EmbeddedData){')
+	g.writeln('(v__embed_file__EmbedFileData){')
-	g.writeln('\t.path = ${ctoslit(node.embed_file.rpath)},')
+	g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},')
-	g.writeln('\t.apath = ${ctoslit(node.embed_file.apath)},')
+	g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},')
 	file_size := os.file_size(node.embed_file.apath)
 	if file_size > 5242880 {
 		eprintln('Warning: embedding of files >= ~5MB is currently not supported')
 	}
-	if g.pref.is_prod {
+	if g.embed_file_is_prod_mode() {
 		// Use function generated in Gen.gen_embedded_data()
-		g.writeln('\t.compressed = _v_embed_locate_data(${ctoslit(node.embed_file.apath)}),')
+		g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)})->data,')
 	}
-	g.writeln('\t.len = $file_size')
+	g.writeln('\t\t.uncompressed = NULL,')
+	g.writeln('\t\t.free_compressed = 0,')
+	g.writeln('\t\t.free_uncompressed = 0,')
+	g.writeln('\t\t.len = $file_size')
 	g.writeln('} // $' + 'embed_file("$node.embed_file.apath")')
 }
 
@@ -49,27 +59,11 @@ fn (mut g Gen) gen_embedded_data() {
 		g.embedded_data.writeln('\n};')
 	}
 	g.embedded_data.writeln('')
-	g.embedded_data.writeln('const struct _v_embed {')
+	g.embedded_data.writeln('const v__embed_file__EmbedFileIndexEntry _v_embed_file_index[] = {')
-	g.embedded_data.writeln('\tstring id;')
-	g.embedded_data.writeln('\tbyteptr data;')
-	g.embedded_data.writeln('}')
-	g.embedded_data.writeln('_v_embedded_data[] = {')
 	for i, emfile in g.embedded_files {
-		g.embedded_data.writeln('\t{${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
+		g.embedded_data.writeln('\t{$i, ${ctoslit(emfile.rpath)}, _v_embed_blob_$i},')
 	}
-	g.embedded_data.writeln('\t{_SLIT(""), NULL}')
+	g.embedded_data.writeln('\t{-1, _SLIT(""), NULL}')
 	g.embedded_data.writeln('};')
-	// See `vlib/v/gen/comptime.v` -> Gen.comptime_call_embed_file(), where this is called at runtime.
+	// see vlib/v/embed_file/embed_file.v, find_index_entry_by_id/2 and find_index_entry_by_path/2
-	// Generate function to locate the data.
-	g.embedded_data.writeln('
-// function to locate embedded data by a vstring
-byteptr _v_embed_locate_data(string id) {
-	const struct _v_embed *ve;
-	for (ve = _v_embedded_data; !string_eq(ve->id, _SLIT("")) && ve->data != NULL; ve++) {
-		if (string_eq(ve->id, id)) {
-			return (byteptr) ve->data;
-		}
-	}
-	return NULL;
-}')
 }

vlib/v/live/executable/reloader.v

@@ -11,7 +11,6 @@ pub const (
 )
 
 // The live reloader code is implemented here.
-
 // NB: new_live_reload_info will be called by generated C code inside main()
 pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_mutex voidptr, live_linkfn live.FNLinkLiveSymbols) &live.LiveReloadInfo {
 	file_base := os.file_name(original).replace('.v', '')
@@ -28,7 +27,7 @@ pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_
 		live_fn_mutex: live_fn_mutex
 		live_linkfn: live_linkfn
 		so_extension: so_extension
-		so_name_template: '${so_dir}/tmp.%d.${file_base}'
+		so_name_template: '$so_dir/tmp.%d.$file_base'
 		live_lib: 0
 		reloads: 0
 		reload_time_ms: 0
@@ -44,13 +43,13 @@ pub fn start_reloader(mut r live.LiveReloadInfo) {
 	// an error message to the user and exit:
 	r.reloads++
 	compile_and_reload_shared_lib(mut r) or {
-		eprintln( err )
+		eprintln(err)
 		exit(1)
 	}
 	go reloader(mut r)
 }
 
-fn elog(r &live.LiveReloadInfo, s string){
+fn elog(r &live.LiveReloadInfo, s string) {
 	$if debuglive ? {
 		eprintln(s)
 	}
@@ -58,11 +57,9 @@ fn elog(r &live.LiveReloadInfo, s string){
 
 fn compile_and_reload_shared_lib(mut r live.LiveReloadInfo) ?bool {
 	sw := time.new_stopwatch({})
-	new_lib_path := compile_lib(mut r) or {
+	new_lib_path := compile_lib(mut r) or { return error('errors while compiling $r.original') }
-		return error('errors while compiling $r.original')
+	elog(r, '> compile_and_reload_shared_lib compiled: $new_lib_path')
-	}
+	load_lib(mut r, new_lib_path)
-	elog(r,'> compile_and_reload_shared_lib compiled: ${new_lib_path}')
-	load_lib(mut r, new_lib_path )
 	r.reload_time_ms = int(sw.elapsed().milliseconds())
 	return true
 }
@@ -70,19 +67,19 @@ fn compile_and_reload_shared_lib(mut r live.LiveReloadInfo) ?bool {
 fn compile_lib(mut r live.LiveReloadInfo) ?string {
 	new_lib_path, new_lib_path_with_extension := current_shared_library_path(mut r)
 	cmd := '$r.vexe $r.vopts -o $new_lib_path $r.original'
-	elog(r,'>       compilation cmd: $cmd')
+	elog(r, '>       compilation cmd: $cmd')
 	cwatch := time.new_stopwatch({})
-	recompilation_result := os.exec( cmd ) or {
+	recompilation_result := os.exec(cmd) or {
 		eprintln('recompilation failed')
 		return none
 	}
-	elog(r,'compilation took: ${cwatch.elapsed().milliseconds()}ms')
+	elog(r, 'compilation took: ${cwatch.elapsed().milliseconds()}ms')
 	if recompilation_result.exit_code != 0 {
 		eprintln('recompilation error:')
-		eprintln( recompilation_result.output )
+		eprintln(recompilation_result.output)
 		return none
 	}
-	if !os.exists( new_lib_path_with_extension ) {
+	if !os.exists(new_lib_path_with_extension) {
 		eprintln('new_lib_path: $new_lib_path_with_extension does not exist')
 		return none
 	}
@@ -96,29 +93,29 @@ fn current_shared_library_path(mut r live.LiveReloadInfo) (string, string) {
 }
 
 fn load_lib(mut r live.LiveReloadInfo, new_lib_path string) {
-	elog(r,'live mutex locking...')
+	elog(r, 'live mutex locking...')
 	C.pthread_mutex_lock(r.live_fn_mutex)
-	elog(r,'live mutex locked')
+	elog(r, 'live mutex locked')
 	//
 	if r.cb_locked_before != voidptr(0) {
-		r.cb_locked_before( r )
+		r.cb_locked_before(r)
 	}
 	//
 	protected_load_lib(mut r, new_lib_path)
 	//
 	r.reloads_ok++
 	if r.cb_locked_after != voidptr(0) {
-		r.cb_locked_after( r )
+		r.cb_locked_after(r)
 	}
 	//
-	elog(r,'live mutex unlocking...')
+	elog(r, 'live mutex unlocking...')
 	C.pthread_mutex_unlock(r.live_fn_mutex)
-	elog(r,'live mutex unlocked')
+	elog(r, 'live mutex unlocked')
 }
 
 fn protected_load_lib(mut r live.LiveReloadInfo, new_lib_path string) {
 	if r.live_lib != 0 {
-		dl.close( r.live_lib )
+		dl.close(r.live_lib)
 		r.live_lib = C.NULL
 	}
 	r.live_lib = dl.open(new_lib_path, dl.rtld_lazy)
@@ -126,40 +123,40 @@ fn protected_load_lib(mut r live.LiveReloadInfo, new_lib_path string) {
 		eprintln('opening $new_lib_path failed')
 		exit(1)
 	}
-	r.live_linkfn( r.live_lib )
+	r.live_linkfn(r.live_lib)
-	elog(r,'> load_lib OK, new live_lib: $r.live_lib')
+	elog(r, '> load_lib OK, new live_lib: $r.live_lib')
 	// removing the .so file from the filesystem after dlopen-ing
 	// it is safe, since it will still be mapped in memory
-	os.rm( new_lib_path )
+	os.rm(new_lib_path)
 }
 
 // NB: r.reloader() is executed in a new, independent thread
 fn reloader(mut r live.LiveReloadInfo) {
-//	elog(r,'reloader, r: $r')
+	//	elog(r,'reloader, r: $r')
-	mut last_ts := os.file_last_mod_unix( r.original )
+	mut last_ts := os.file_last_mod_unix(r.original)
 	for {
 		if r.cb_recheck != voidptr(0) {
-			r.cb_recheck( r )
+			r.cb_recheck(r)
 		}
-		now_ts := os.file_last_mod_unix( r.original )
+		now_ts := os.file_last_mod_unix(r.original)
 		if last_ts != now_ts {
 			r.reloads++
 			last_ts = now_ts
 			r.last_mod_ts = last_ts
 			if r.cb_before != voidptr(0) {
-				r.cb_before( r )
+				r.cb_before(r)
 			}
 			compile_and_reload_shared_lib(mut r) or {
 				if r.cb_compile_failed != voidptr(0) {
-					r.cb_compile_failed( r )
+					r.cb_compile_failed(r)
 				}
 				if r.cb_after != voidptr(0) {
-					r.cb_after( r )
+					r.cb_after(r)
 				}
 				continue
 			}
 			if r.cb_after != voidptr(0) {
-				r.cb_after( r )
+				r.cb_after(r)
 			}
 		}
 		if r.recheck_period_ms > 0 {

vlib/v/live/live_test.v

@@ -44,9 +44,7 @@ const (
 )
 
 fn get_source_template() string {
-	src := os.read_file(os.join_path(os.dir(@FILE), 'live_test_template.vv')) or {
+	src := os.read_file(os.join_path(os.dir(@FILE), 'live_test_template.vv')) or { panic(err) }
-    	panic(err)
-    }
 	return src.replace('#OUTPUT_FILE#', output_file)
 }
 

vlib/v/parser/comptime.v

@@ -95,7 +95,7 @@ fn (mut p Parser) comp_call() ast.ComptimeCall {
 				epath = abs_path
 			}
 		}
-		p.register_auto_import('embed')
+		p.register_auto_import('v.embed_file')
 		return ast.ComptimeCall{
 			is_embed: true
 			embed_file: ast.EmbeddedFile{

vlib/v/pkgconfig/pkgconfig.v

@@ -43,7 +43,7 @@ pub mut:
 }
 
 fn (mut pc PkgConfig) parse_list_no_comma(s string) []string {
-	return pc.parse_list( s.replace(',', ' ') )
+	return pc.parse_list(s.replace(',', ' '))
 }
 
 fn (mut pc PkgConfig) parse_list(s string) []string {

vlib/v/pref/pref.v

@@ -114,7 +114,7 @@ pub mut:
 	display_name   string
 	bundle_id      string
 	path           string // Path to file/folder to compile
-	// -d vfmt and -d another=0 for `$if vfmt { will execute }` and `$if another { will NOT get here }`
+	// -d vfmt and -d another=0 for `$if vfmt { will execute }` and `$if another ? { will NOT get here }`
 	compile_defines     []string    // just ['vfmt']
 	compile_defines_all []string    // contains both: ['vfmt','another']
 	run_args            []string    // `v run x.v 1 2 3` => `1 2 3`