all: split all backends into cmd/tools/builders (#12811)

cmd/tools/builders/c_builder.v

@@ -0,0 +1,11 @@
+module main
+
+import v.builder.cbuilder
+
+// TODO: change bootstrapping to use the C code generated from
+// `VEXE=v cmd/tools/builders/c_builder -os cross -o c.c cmd/tools/builders/c_builder.v`
+// See also `cmd/v/v.v`
+
+fn main() {
+	cbuilder.start()
+}

cmd/tools/builders/interpret_builder.v

@@ -0,0 +1,7 @@
+module main
+
+import v.builder.interpreterbuilder
+
+fn main() {
+	interpreterbuilder.start()
+}

cmd/tools/builders/js_builder.v

@@ -0,0 +1,7 @@
+module main
+
+import v.builder.jsbuilder
+
+fn main() {
+	jsbuilder.start()
+}

cmd/tools/builders/native_builder.v

@@ -0,0 +1,7 @@
+module main
+
+import v.builder.nativebuilder
+
+fn main() {
+	nativebuilder.start()
+}

cmd/tools/check_os_api_parity.v

@@ -5,6 +5,7 @@ import v.util
 import v.util.diff
 import v.pref
 import v.builder
+import v.builder.cbuilder
 import v.ast
 import rand
 import term
@@ -98,7 +99,7 @@ fn (app App) gen_api_for_module_in_os(mod_name string, os_name string) string {
 	tmpname := '/tmp/${mod_name}_${os_name}.c'
 	prefs, _ := pref.parse_args([], ['-os', os_name, '-o', tmpname, '-shared', mpath])
 	mut b := builder.new_builder(prefs)
-	builder.compile_c(mut b)
+	cbuilder.compile_c(mut b)
 	mut res := []string{}
 	for f in b.parsed_files {
 		for s in f.stmts {

cmd/tools/vbuild-tools.v

@@ -56,6 +56,10 @@ fn main() {
 		}
 		//
 		tpath := os.join_path(session.vtmp_dir, texe)
+		if texe.ends_with('_builder') || texe.ends_with('_builder.exe') {
+			os.mv_by_cp(tpath, os.join_path(tfolder, 'builders', texe)) or { panic(err) }
+			continue
+		}
 		if tname in tools_in_subfolders {
 			os.mv_by_cp(tpath, os.join_path(tfolder, tname, texe)) or { panic(err) }
 			continue

cmd/v/v.v

@@ -10,6 +10,7 @@ import v.pref
 import v.util
 import v.util.version
 import v.builder
+import v.builder.cbuilder
 
 const (
 	external_tools                      = [
@@ -27,7 +28,6 @@ const (
 		'doctor',
 		'fmt',
 		'gret',
-		'interpret',
 		'repl',
 		'self',
 		'setup-freetype',
@@ -76,14 +76,15 @@ fn main() {
 			} else {
 				mut args_and_flags := util.join_env_vflags_and_os_args()[1..].clone()
 				args_and_flags << ['run', '-']
-				pref.parse_args(external_tools, args_and_flags)
+				pref.parse_args_and_show_errors(external_tools, args_and_flags, true)
 			}
 		}
 		util.launch_tool(false, 'vrepl', os.args[1..])
 		return
 	}
 	mut args_and_flags := util.join_env_vflags_and_os_args()[1..]
-	prefs, command := pref.parse_args(external_tools, args_and_flags)
+	prefs, command := pref.parse_args_and_show_errors(external_tools, args_and_flags,
+		true)
 	if prefs.use_cache && os.user_os() == 'windows' {
 		eprintln('-usecache is currently disabled on windows')
 		exit(1)
@@ -115,6 +116,9 @@ fn main() {
 		'vlib-docs' {
 			util.launch_tool(prefs.is_verbose, 'vdoc', ['doc', 'vlib'])
 		}
+		'interpret' {
+			util.launch_tool(prefs.is_verbose, 'builders/interpret_builder', os.args[1..])
+		}
 		'get' {
 			eprintln('V Error: Use `v install` to install modules from vpm.vlang.io')
 			exit(1)
@@ -128,23 +132,27 @@ fn main() {
 	if command in ['run', 'build', 'build-module'] || command.ends_with('.v') || os.exists(command) {
 		// println('command')
 		// println(prefs.path)
-		backend_cb := match prefs.backend {
+		match prefs.backend {
 			.c {
-				builder.FnBackend(builder.compile_c)
+				$if no_bootstrapv ? {
+					// TODO: improve the bootstrapping with a split C backend here.
+					// C code generated by `VEXE=v cmd/tools/builders/c_builder -os cross -o c.c cmd/tools/builders/c_builder.v`
+					// is enough to bootstrap the C backend, and thus the rest, but currently bootstrapping relies on
+					// `v -os cross -o v.c cmd/v` having a functional C codegen inside instead.
+					util.launch_tool(prefs.is_verbose, 'builders/c_builder', os.args[1..])
+				}
+				builder.compile('build', prefs, cbuilder.compile_c)
 			}
 			.js_node, .js_freestanding, .js_browser {
-				builder.compile_js
+				util.launch_tool(prefs.is_verbose, 'builders/js_builder', os.args[1..])
 			}
 			.native {
-				builder.compile_native
+				util.launch_tool(prefs.is_verbose, 'builders/native_builder', os.args[1..])
 			}
 			.interpret {
-				eprintln('use `v interpret file.v`')
+				util.launch_tool(prefs.is_verbose, 'builders/interpret_builder', os.args[1..])
-				exit(1)
-				builder.compile_c
 			}
 		}
-		builder.compile(command, prefs, backend_cb)
 		return
 	}
 	if prefs.is_help {

vlib/v/ast/native.v

@@ -0,0 +1,7 @@
+module ast
+
+// This file contains definitions that are specific to the native backend,
+// but also have to be known by previous stages too, like the parser/checker etc.
+// Please keep it as small/simple as possible, in order to not burden the *other* backends.
+
+pub const native_builtins = ['assert', 'print', 'eprint', 'println', 'eprintln', 'exit', 'C.syscall']

vlib/v/builder/builder.v

@@ -19,7 +19,7 @@ pub struct Builder {
 pub:
 	compiled_dir string // contains os.real_path() of the dir of the final file beeing compiled, or the dir itself when doing `v .`
 	module_path  string
-mut:
+pub mut:
 	checker             &checker.Checker
 	transformer         &transformer.Transformer
 	out_name_c          string
@@ -29,7 +29,6 @@ mut:
 	nr_errors           int // accumulated error count of scanner, parser, checker, and builder
 	nr_warnings         int // accumulated warning count of scanner, parser, checker, and builder
 	nr_notices          int // accumulated notice count of scanner, parser, checker, and builder
-pub mut:
 	pref                &pref.Preferences
 	module_search_paths []string
 	parsed_files        []&ast.File
@@ -319,7 +318,7 @@ pub fn (b Builder) info(s string) {
 }
 
 [inline]
-fn module_path(mod string) string {
+pub fn module_path(mod string) string {
 	// submodule support
 	return mod.replace('.', os.path_separator)
 }
@@ -377,7 +376,7 @@ pub fn (b &Builder) find_module_path(mod string, fpath string) ?string {
 	return error('module "$mod" not found in:\n$smodule_lookup_paths')
 }
 
-fn (b &Builder) show_total_warns_and_errors_stats() {
+pub fn (b &Builder) show_total_warns_and_errors_stats() {
 	if b.nr_errors == 0 && b.nr_warnings == 0 && b.nr_notices == 0 {
 		return
 	}
@@ -404,7 +403,7 @@ fn (b &Builder) show_total_warns_and_errors_stats() {
 	}
 }
 
-fn (mut b Builder) print_warnings_and_errors() {
+pub fn (mut b Builder) print_warnings_and_errors() {
 	defer {
 		b.show_total_warns_and_errors_stats()
 	}
@@ -578,7 +577,7 @@ struct FunctionRedefinition {
 	f       ast.FnDecl
 }
 
-fn (b &Builder) error_with_pos(s string, fpath string, pos token.Position) errors.Error {
+pub fn (b &Builder) error_with_pos(s string, fpath string, pos token.Position) errors.Error {
 	if !b.pref.check_only {
 		ferror := util.formatted_error('builder error:', s, fpath, pos)
 		eprintln(ferror)
@@ -594,6 +593,6 @@ fn (b &Builder) error_with_pos(s string, fpath string, pos token.Position) error
 }
 
 [noreturn]
-fn verror(s string) {
+pub fn verror(s string) {
 	util.verror('builder error', s)
 }

vlib/v/builder/cbuilder/cbuilder.v

@@ -1,26 +1,24 @@
-module builder
+module cbuilder
 
 import os
 import v.pref
 import v.util
+import v.builder
 import v.gen.c
 
-pub fn compile_c(mut b Builder) {
+pub fn start() {
-	// cgen.genln('// Generated by V')
+	mut args_and_flags := util.join_env_vflags_and_os_args()[1..]
-	// println('compile2()')
+	prefs, _ := pref.parse_args([], args_and_flags)
+	builder.compile('build', prefs, compile_c)
+}
+
+pub fn compile_c(mut b builder.Builder) {
 	if b.pref.is_verbose {
 		println('all .v files before:')
-		// println(files)
 	}
 	$if windows {
-		b.find_win_cc() or { verror(no_compiler_error) }
+		b.find_win_cc() or { builder.verror(builder.no_compiler_error) }
-		// TODO Probably extend this to other OS's?
 	}
-	// v1 compiler files
-	// v.add_v_files_to_compile()
-	// v.files << v.dir
-	// v2 compiler
-	// b.set_module_lookup_paths()
 	mut files := b.get_builtin_files()
 	files << b.get_user_files()
 	b.set_module_lookup_paths()
@@ -32,31 +30,28 @@ pub fn compile_c(mut b Builder) {
 	if b.pref.is_shared {
 		out_name_c = b.get_vtmp_filename(b.pref.out_name, '.tmp.so.c')
 	}
-	b.build_c(files, out_name_c)
+	build_c(mut b, files, out_name_c)
 	b.cc()
 }
 
-pub fn (mut b Builder) gen_c(v_files []string) string {
+pub fn gen_c(mut b builder.Builder, v_files []string) string {
 	b.front_and_middle_stages(v_files) or {
 		if err.code != 9999 {
-			verror(err.msg)
+			builder.verror(err.msg)
 		}
 		return ''
 	}
-	// TODO: move gen.cgen() to c.gen()
 	util.timing_start('C GEN')
 	res := c.gen(b.parsed_files, b.table, b.pref)
 	util.timing_measure('C GEN')
-	// println('cgen done')
-	// println(res)
 	return res
 }
 
-pub fn (mut b Builder) build_c(v_files []string, out_file string) {
+pub fn build_c(mut b builder.Builder, v_files []string, out_file string) {
 	b.out_name_c = out_file
 	b.pref.out_name_c = os.real_path(out_file)
 	b.info('build_c($out_file)')
-	output2 := b.gen_c(v_files)
+	output2 := gen_c(mut b, v_files)
 	os.write_file(out_file, output2) or { panic(err) }
 	if b.pref.is_stats {
 		b.stats_lines = output2.count('\n') + 1

vlib/v/builder/cc.v

@@ -10,9 +10,9 @@ import v.util
 import v.vcache
 import term
 
-const (
+const c_verror_message_marker = 'VERROR_MESSAGE '
-	c_verror_message_marker = 'VERROR_MESSAGE '
+
-	c_error_info            = '
+const c_error_info = '
 ==================
 C error. This should never happen.
 
@@ -22,7 +22,8 @@ https://github.com/vlang/v/issues/new/choose
 
 You can also use #help on Discord: https://discord.gg/vlang
 '
-	no_compiler_error = '
+
+pub const no_compiler_error = '
 ==================
 Error: no C compiler detected.
 
@@ -37,9 +38,8 @@ You can also use `v doctor`, to see what V knows about your current environment.
 
 You can also seek #help on Discord: https://discord.gg/vlang
 '
-)
 
-fn (mut v Builder) find_win_cc() ? {
+pub fn (mut v Builder) find_win_cc() ? {
 	$if !windows {
 		return
 	}
@@ -443,7 +443,7 @@ fn (mut v Builder) dump_c_options(all_args []string) {
 	}
 }
 
-fn (mut v Builder) cc() {
+pub fn (mut v Builder) cc() {
 	if os.executable().contains('vfmt') {
 		return
 	}

vlib/v/builder/compile.v

@@ -66,7 +66,7 @@ pub fn compile(command string, pref &pref.Preferences, backend_cb FnBackend) {
 	}
 }
 
-fn (mut b Builder) get_vtmp_filename(base_file_name string, postfix string) string {
+pub fn (mut b Builder) get_vtmp_filename(base_file_name string, postfix string) string {
 	vtmp := util.get_vtmp_folder()
 	mut uniq := ''
 	if !b.pref.reuse_tmpc {

vlib/v/builder/interpreterbuilder/ibuilder.v

@@ -1,17 +1,17 @@
-module main
+module interpreterbuilder
 
-import v.pref
 import v.eval
+import v.pref
 import v.util
 import v.builder
 
-fn main() {
+pub fn start() {
-	mut args_and_flags := util.join_env_vflags_and_os_args()[1..].filter(it != 'interpret')
+	mut args_and_flags := util.join_env_vflags_and_os_args()[1..]
 	prefs, _ := pref.parse_args([], args_and_flags)
-	builder.compile('interpret', prefs, v_interpret)
+	builder.compile('interpret', prefs, interpret_v)
 }
 
-fn v_interpret(mut b builder.Builder) {
+pub fn interpret_v(mut b builder.Builder) {
 	mut files := b.get_builtin_files()
 	files << b.get_user_files()
 	b.set_module_lookup_paths()

vlib/v/builder/jsbuilder/jsbuilder.v

@@ -1,11 +1,18 @@
-module builder
+module jsbuilder
 
 import os
 import v.pref
 import v.util
+import v.builder
 import v.gen.js
 
-pub fn compile_js(mut b Builder) {
+pub fn start() {
+	mut args_and_flags := util.join_env_vflags_and_os_args()[1..]
+	prefs, _ := pref.parse_args([], args_and_flags)
+	builder.compile('build', prefs, compile_js)
+}
+
+pub fn compile_js(mut b builder.Builder) {
 	mut files := b.get_builtin_files()
 	files << b.get_user_files()
 	b.set_module_lookup_paths()
@@ -17,21 +24,13 @@ pub fn compile_js(mut b Builder) {
 	if !name.ends_with('.js') {
 		name += '.js'
 	}
-	b.build_js(files, name)
+	build_js(mut b, files, name)
 }
 
-pub fn (mut b Builder) gen_js(v_files []string) string {
+pub fn build_js(mut b builder.Builder, v_files []string, out_file string) {
-	b.front_and_middle_stages(v_files) or { return '' }
-	util.timing_start('JS GEN')
-	res := js.gen(b.parsed_files, b.table, b.pref)
-	util.timing_measure('JS GEN')
-	return res
-}
-
-pub fn (mut b Builder) build_js(v_files []string, out_file string) {
 	b.out_name_js = out_file
 	b.info('build_js($out_file)')
-	output := b.gen_js(v_files)
+	output := gen_js(mut b, v_files)
 	os.write_file(out_file, output) or { panic(err) }
 	if b.pref.is_stats {
 		b.stats_lines = output.count('\n') + 1
@@ -39,13 +38,10 @@ pub fn (mut b Builder) build_js(v_files []string, out_file string) {
 	}
 }
 
-fn (mut b Builder) run_js() {
+pub fn gen_js(mut b builder.Builder, v_files []string) string {
-	cmd := 'node ' + b.pref.out_name + '.js'
+	b.front_and_middle_stages(v_files) or { return '' }
-	res := os.execute(cmd)
+	util.timing_start('JS GEN')
-	if res.exit_code != 0 {
+	res := js.gen(b.parsed_files, b.table, b.pref)
-		eprintln('JS compilation failed:')
+	util.timing_measure('JS GEN')
-		verror(res.output)
+	return res
-		return
-	}
-	println(res.output)
 }

vlib/v/builder/nativebuilder/nativebuilder.v

@@ -1,18 +1,25 @@
-module builder
+module nativebuilder
 
+import os
 import v.pref
 import v.util
+import v.builder
 import v.gen.native
-import os
 
-pub fn compile_native(mut b Builder) {
+pub fn start() {
+	mut args_and_flags := util.join_env_vflags_and_os_args()[1..]
+	prefs, _ := pref.parse_args([], args_and_flags)
+	builder.compile('build', prefs, compile_native)
+}
+
+pub fn compile_native(mut b builder.Builder) {
 	// v.files << v.v_files_from_dir(os.join_path(v.pref.vlib_path,'builtin','bare'))
 	files := [b.pref.path]
 	b.set_module_lookup_paths()
-	b.build_native(files, b.pref.out_name)
+	build_native(mut b, files, b.pref.out_name)
 }
 
-pub fn (mut b Builder) build_native(v_files []string, out_file string) {
+pub fn build_native(mut b builder.Builder, v_files []string, out_file string) {
 	if b.pref.os == .windows {
 		eprintln('Warning: v -native is experimental for Windows')
 		if !b.pref.is_shared && b.pref.build_mode != .build_module

vlib/v/checker/fn.v

@@ -3,7 +3,6 @@ module checker
 import v.ast
 import v.pref
 import time
-import v.gen.native
 import v.util
 
 fn (mut c Checker) fn_decl(mut node ast.FnDecl) {
@@ -497,7 +496,7 @@ pub fn (mut c Checker) fn_call(mut node ast.CallExpr, mut continue_check &bool)
 	}
 	mut is_native_builtin := false
 	if !found && c.pref.backend == .native {
-		if fn_name in native.builtins {
+		if fn_name in ast.native_builtins {
 			c.table.fns[fn_name].usages++
 			found = true
 			func = c.table.fns[fn_name]

vlib/v/pref/pref.v

@@ -197,6 +197,10 @@ pub mut:
 }
 
 pub fn parse_args(known_external_commands []string, args []string) (&Preferences, string) {
+	return parse_args_and_show_errors(known_external_commands, args, false)
+}
+
+pub fn parse_args_and_show_errors(known_external_commands []string, args []string, show_output bool) (&Preferences, string) {
 	mut res := &Preferences{}
 	$if x64 {
 		res.m64 = true // follow V model by default
@@ -374,7 +378,7 @@ pub fn parse_args(known_external_commands []string, args []string) (&Preferences
 				res.is_shared = true
 			}
 			'--enable-globals' {
-				eprintln('`--enable-globals` flag is deprecated, please use `-enable-globals` instead')
+				eprintln_cond(show_output, '`--enable-globals` flag is deprecated, please use `-enable-globals` instead')
 				res.enable_globals = true
 			}
 			'-enable-globals' {
@@ -642,8 +646,8 @@ pub fn parse_args(known_external_commands []string, args []string) (&Preferences
 		res.parse_define('debug')
 	}
 	if command == 'run' && res.is_prod && os.is_atty(1) > 0 {
-		eprintln("NB: building an optimized binary takes much longer. It shouldn't be used with `v run`.")
+		eprintln_cond(show_output, "NB: building an optimized binary takes much longer. It shouldn't be used with `v run`.")
-		eprintln('Use `v run` without optimization, or build an optimized binary with -prod first, then run it separately.')
+		eprintln_cond(show_output, 'Use `v run` without optimization, or build an optimized binary with -prod first, then run it separately.')
 	}
 
 	// res.use_cache = true
@@ -692,14 +696,14 @@ pub fn parse_args(known_external_commands []string, args []string) (&Preferences
 		must_exist(res.path)
 		if !res.path.ends_with('.v') && os.is_executable(res.path) && os.is_file(res.path)
 			&& os.is_file(res.path + '.v') {
-			eprintln('It looks like you wanted to run "${res.path}.v", so we went ahead and did that since "$res.path" is an executable.')
+			eprintln_cond(show_output, 'It looks like you wanted to run "${res.path}.v", so we went ahead and did that since "$res.path" is an executable.')
 			res.path += '.v'
 		}
 	} else if is_source_file(command) {
 		res.path = command
 	}
 	if !res.is_bare && res.bare_builtin_dir != '' {
-		eprintln('`-bare-builtin-dir` must be used with `-freestanding`')
+		eprintln_cond(show_output, '`-bare-builtin-dir` must be used with `-freestanding`')
 	}
 	if command.ends_with('.vsh') {
 		// `v build.vsh gcc` is the same as `v run build.vsh gcc`,
@@ -743,6 +747,13 @@ pub fn parse_args(known_external_commands []string, args []string) (&Preferences
 	return res, command
 }
 
+pub fn eprintln_cond(condition bool, s string) {
+	if !condition {
+		return
+	}
+	eprintln(s)
+}
+
 pub fn (pref &Preferences) vrun_elog(s string) {
 	if pref.is_verbose {
 		eprintln('> v run -, $s')

vlib/v/util/util.v

@@ -115,6 +115,7 @@ pub fn resolve_env_value(str string, check_for_presence bool) ?string {
 // V itself. That mechanism can be disabled by package managers by creating/touching a small
 // `cmd/tools/.disable_autorecompilation` file, OR by changing the timestamps of all executables
 // in cmd/tools to be < 1024 seconds (in unix time).
+[noreturn]
 pub fn launch_tool(is_verbose bool, tool_name string, args []string) {
 	vexe := pref.vexe_path()
 	vroot := os.dir(vexe)
@@ -125,7 +126,7 @@ pub fn launch_tool(is_verbose bool, tool_name string, args []string) {
 	mut tool_exe := ''
 	mut tool_source := ''
 	if os.is_dir(tool_basename) {
-		tool_exe = path_of_executable(os.join_path_single(tool_basename, tool_name))
+		tool_exe = path_of_executable(os.join_path_single(tool_basename, os.file_name(tool_name)))
 		tool_source = tool_basename
 	} else {
 		tool_exe = path_of_executable(tool_basename)
@@ -177,6 +178,7 @@ pub fn launch_tool(is_verbose bool, tool_name string, args []string) {
 	} $else {
 		os.execvp(tool_exe, args) or { panic(err) }
 	}
+	exit(2)
 }
 
 // NB: should_recompile_tool/4 compares unix timestamps that have 1 second resolution