From 2d5c70832c1bcb21b315e3607371e1e4f4c843ee Mon Sep 17 00:00:00 2001
From: joe-conigliaro <joe-conigliaro@users.noreply.github.com>
Date: Mon, 3 Feb 2020 17:31:54 +1100
Subject: [PATCH] v2: initial module support

---
 vlib/compiler/aparser.v  |   3 +-
 vlib/compiler/main.v     |  39 +++++------
 vlib/v/ast/ast.v         |   1 +
 vlib/v/builder/builder.v | 139 ++++++++++++++++++++++++++++++++++++---
 vlib/v/builder/modules.v |  59 +++++++++++++++++
 vlib/v/builder/prefs.v   |  30 +++++++++
 vlib/v/checker/checker.v |   6 +-
 vlib/v/gen/cgen_test.v   |   2 +-
 vlib/v/gen/tests/1.vv    |   4 +-
 vlib/v/parser/parser.v   |   1 +
 10 files changed, 250 insertions(+), 34 deletions(-)
 create mode 100644 vlib/v/builder/modules.v
 create mode 100644 vlib/v/builder/prefs.v

diff --git a/vlib/compiler/aparser.v b/vlib/compiler/aparser.v
index e96ccbfb09..d1aaee7850 100644
--- a/vlib/compiler/aparser.v
+++ b/vlib/compiler/aparser.v
@@ -7,6 +7,7 @@ import (
 	os
 	strings
 	filepath
+	v.builder
 	//compiler.x64
 	time
 )
@@ -35,7 +36,7 @@ mut:
 	table                  &Table
 	import_table           ImportTable // Holds imports for just the file being parsed
 	pass                   Pass
-	os                     OS
+	os                     builder.OS
 	inside_const           bool
 	expr_var               Var
 	has_immutable_field    bool
diff --git a/vlib/compiler/main.v b/vlib/compiler/main.v
index 62c0db8e1b..d79223a733 100644
--- a/vlib/compiler/main.v
+++ b/vlib/compiler/main.v
@@ -29,20 +29,6 @@ const (
 		'dragonfly', 'android', 'js', 'solaris', 'haiku', 'linux_or_macos']
 )
 
-enum OS {
-	mac
-	linux
-	windows
-	freebsd
-	openbsd
-	netbsd
-	dragonfly
-	js // TODO
-	android
-	solaris
-	haiku
-}
-
 enum Pass {
 	// A very short pass that only looks at imports in the beginning of
 	// each file
@@ -59,7 +45,7 @@ enum Pass {
 
 struct V {
 pub mut:
-	os                  OS // the OS to build for
+	os                  builder.OS // the OS to build for
 	out_name_c          string // name of the temporary C file
 	files               []string // all V files that need to be parsed and compiled
 	dir                 string // directory (or file) being compiled (TODO rename to path?)
@@ -393,7 +379,7 @@ pub fn (v mut V) compile2() {
 		println('all .v files:')
 		println(v.files)
 	}
-	mut b := builder.new_builder()
+	mut b := builder.new_builder(v.v2_prefs())
 	b.build_c(v.files, v.out_name)
 	v.cc()
 }
@@ -405,10 +391,25 @@ pub fn (v mut V) compile_x64() {
 	}
 	//v.files << v.v_files_from_dir(filepath.join(v.pref.vlib_path,'builtin','bare'))
 	v.files << v.dir
-	mut b := builder.new_builder()
+	mut b := builder.new_builder(v.v2_prefs())
+	// move all this logic to v2
 	b.build_x64(v.files, v.out_name)
 }
 
+// make v2 prefs from v1
+fn (v &V) v2_prefs() builder.Preferences {
+	return builder.Preferences{
+		os: v.os
+		vpath: v.pref.vpath
+		vlib_path: v.pref.vlib_path
+		mod_path: v_modules_path
+		compile_dir: v.compiled_dir
+		user_mod_path: v.pref.user_mod_path
+		is_test: v.pref.is_test
+		is_verbose: v.pref.is_verbose,
+	}
+}
+
 fn (v mut V) generate_init() {
 	$if js {
 		return
@@ -1013,7 +1014,7 @@ pub fn new_v(args []string) &V {
 			}
 		}
 	}
-	mut _os := OS.mac
+	mut _os := builder.OS.mac
 	// No OS specifed? Use current system
 	if target_os == '' {
 		$if linux {
@@ -1224,7 +1225,7 @@ pub fn cescaped_path(s string) string {
 	return s.replace('\\', '\\\\')
 }
 
-pub fn os_from_string(os string) OS {
+pub fn os_from_string(os string) builder.OS {
 	match os {
 		'linux' {
 			return .linux
diff --git a/vlib/v/ast/ast.v b/vlib/v/ast/ast.v
index 1c3cf2b9f7..23f71dd9bf 100644
--- a/vlib/v/ast/ast.v
+++ b/vlib/v/ast/ast.v
@@ -175,6 +175,7 @@ mut:
 
 pub struct File {
 pub:
+	path    string
 	mod     Module
 	imports []Import
 	stmts   []Stmt
diff --git a/vlib/v/builder/builder.v b/vlib/v/builder/builder.v
index 7681bf5d71..2d8b61fbe4 100644
--- a/vlib/v/builder/builder.v
+++ b/vlib/v/builder/builder.v
@@ -3,6 +3,8 @@ module builder
 import (
 	os
 	time
+	filepath
+	v.ast
 	v.table
 	v.checker
 	v.parser
@@ -14,20 +16,27 @@ pub struct Builder {
 pub:
 	table   &table.Table
 	checker checker.Checker
+mut:
+	prefs   Preferences
+	parsed_files []ast.File
 }
 
-pub fn new_builder() Builder {
+pub fn new_builder(prefs Preferences) Builder {
 	table := table.new_table()
-	return Builder{
+	mut b:= Builder{
+		prefs: prefs
 		table: table
 		checker: checker.new_checker(table)
 	}
+	b.set_module_search_paths()
+	return b
 }
 
 pub fn (b mut Builder) gen_c(v_files []string) string {
-	ast_files := parser.parse_files(v_files, b.table)
-	b.checker.check_files(v_files, ast_files)
-	return gen.cgen(ast_files, b.table)
+	b.parsed_files = parser.parse_files(v_files, b.table)
+	b.parse_imports()
+	b.checker.check_files(b.parsed_files)
+	return gen.cgen(b.parsed_files, b.table)
 }
 
 pub fn (b mut Builder) build_c(v_files []string, out_file string) {
@@ -36,10 +45,124 @@ pub fn (b mut Builder) build_c(v_files []string, out_file string) {
 
 pub fn (b mut Builder) build_x64(v_files []string, out_file string) {
 	ticks := time.ticks()
-	ast_files := parser.parse_files(v_files, b.table)
+	b.parsed_files = parser.parse_files(v_files, b.table)
+	b.parse_imports()
 	println('PARSE: ${time.ticks() - ticks}ms')
-	b.checker.check_files(v_files, ast_files)
+	b.checker.check_files(b.parsed_files)
 	println('CHECK: ${time.ticks() - ticks}ms')
-	x64.gen(ast_files, out_file)
+	x64.gen(b.parsed_files, out_file)
 	println('x64 GEN: ${time.ticks() - ticks}ms')
 }
+
+
+fn (b &Builder) parse_module_files() {
+
+}
+
+// parse all deps from already parsed files
+pub fn (b mut Builder) parse_imports() {
+	mut done_imports := []string
+	for i in 0 .. b.parsed_files.len {
+		ast_file := b.parsed_files[i]
+		for _, imp in ast_file.imports {
+			mod := imp.mod
+			if mod in done_imports {
+				continue
+			}
+			import_path := b.find_module_path(mod) or {
+				//v.parsers[i].error_with_token_index('cannot import module "$mod" (not found)', v.parsers[i].import_table.get_import_tok_idx(mod))
+				//break
+				panic('cannot import module "$mod" (not found)')
+			}
+			v_files := b.v_files_from_dir(import_path)
+			if v_files.len == 0 {
+				//v.parsers[i].error_with_token_index('cannot import module "$mod" (no .v files in "$import_path")', v.parsers[i].import_table.get_import_tok_idx(mod))
+				panic('cannot import module "$mod" (no .v files in "$import_path")')
+			}
+			// Add all imports referenced by these libs
+			parsed_files := parser.parse_files(v_files, b.table)
+			for file in parsed_files {
+				if file.mod.name != mod {
+					//v.parsers[pidx].error_with_token_index('bad module definition: ${v.parsers[pidx].file_path} imports module "$mod" but $file is defined as module `$p_mod`', 1
+					panic('bad module definition: ${ast_file.path} imports module "$mod" but $file.path is defined as module `$ast_file.mod.name`')
+				}
+			}
+			b.parsed_files << parsed_files
+			done_imports << mod
+		}
+	}
+}
+
+pub fn (b &Builder) v_files_from_dir(dir string) []string {
+	mut res := []string
+	if !os.exists(dir) {
+		if dir == 'compiler' && os.is_dir('vlib') {
+			println('looks like you are trying to build V with an old command')
+			println('use `v -o v v.v` instead of `v -o v compiler`')
+		}
+		verror("$dir doesn't exist")
+	}
+	else if !os.is_dir(dir) {
+		verror("$dir isn't a directory")
+	}
+	mut files := os.ls(dir)or{
+		panic(err)
+	}
+	if b.prefs.is_verbose {
+		println('v_files_from_dir ("$dir")')
+	}
+	files.sort()
+	for file in files {
+		if !file.ends_with('.v') && !file.ends_with('.vh') {
+			continue
+		}
+		if file.ends_with('_test.v') {
+			continue
+		}
+		if (file.ends_with('_win.v') || file.ends_with('_windows.v')) && b.prefs.os != .windows {
+			continue
+		}
+		if (file.ends_with('_lin.v') || file.ends_with('_linux.v')) && b.prefs.os != .linux {
+			continue
+		}
+		if (file.ends_with('_mac.v') || file.ends_with('_darwin.v')) && b.prefs.os != .mac {
+			continue
+		}
+		if file.ends_with('_nix.v') && b.prefs.os == .windows {
+			continue
+		}
+		if file.ends_with('_js.v') && b.prefs.os != .js {
+			continue
+		}
+		if file.ends_with('_c.v') && b.prefs.os == .js {
+			continue
+		}
+		/*
+		if v.compile_defines_all.len > 0 && file.contains('_d_') {
+			mut allowed := false
+			for cdefine in v.compile_defines {
+				file_postfix := '_d_${cdefine}.v'
+				if file.ends_with(file_postfix) {
+					allowed = true
+					break
+				}
+			}
+			if !allowed {
+				continue
+			}
+		}
+		*/
+		res << filepath.join(dir,file)
+	}
+	return res
+}
+
+fn verror(err string) {
+	panic('v error: $err')
+}
+
+pub fn (b &Builder) log(s string) {
+	if b.prefs.is_verbose {
+		println(s)
+	}
+}
diff --git a/vlib/v/builder/modules.v b/vlib/v/builder/modules.v
new file mode 100644
index 0000000000..742caa7493
--- /dev/null
+++ b/vlib/v/builder/modules.v
@@ -0,0 +1,59 @@
+module builder
+
+import (
+	os
+	filepath
+)
+
+fn (b mut Builder) set_module_search_paths() {
+	msearch_path := if b.prefs.vpath.len > 0 { b.prefs.vpath } else { b.prefs.mod_path }
+	// Module search order:
+	// 0) V test files are very commonly located right inside the folder of the
+	// module, which they test. Adding the parent folder of the module folder
+	// with the _test.v files, *guarantees* that the tested module can be found
+	// without needing to set custom options/flags.
+	// 1) search in the *same* directory, as the compiled final v program source
+	// (i.e. the . in `v .` or file.v in `v file.v`)
+	// 2) search in the modules/ in the same directory.
+	// 3) search in vlib/
+	// 4.1) search in -vpath (if given)
+	// 4.2) search in ~/.vmodules/ (i.e. modules installed with vpm) (no -vpath)
+	b.prefs.module_search_paths = []
+	if b.prefs.is_test {
+		b.prefs.module_search_paths << filepath.basedir(b.prefs.compile_dir) // pdir of _test.v
+	}
+	b.prefs.module_search_paths << b.prefs.compile_dir
+	b.prefs.module_search_paths << filepath.join(b.prefs.compile_dir,'modules')
+	b.prefs.module_search_paths << b.prefs.vlib_path
+	b.prefs.module_search_paths << msearch_path
+	if b.prefs.user_mod_path.len > 0 {
+		b.prefs.module_search_paths << b.prefs.user_mod_path
+	}
+	if b.prefs.is_verbose {
+		b.log('b.prefs.module_search_paths: $b.prefs.module_search_paths')
+	}
+}
+
+
+[inline]
+fn module_path(mod string) string {
+	// submodule support
+	return mod.replace('.', os.path_separator)
+}
+
+fn (b &Builder) find_module_path(mod string) ?string {
+	mod_path := module_path(mod)
+	for search_path in b.prefs.module_search_paths {
+		try_path := filepath.join(search_path,mod_path)
+		if b.prefs.is_verbose {
+			println('  >> trying to find $mod in $try_path ..')
+		}
+		if os.is_dir(try_path) {
+			if b.prefs.is_verbose {
+				println('  << found $try_path .')
+			}
+			return try_path
+		}
+	}
+	return error('module "$mod" not found')
+}
diff --git a/vlib/v/builder/prefs.v b/vlib/v/builder/prefs.v
new file mode 100644
index 0000000000..6808cd5b05
--- /dev/null
+++ b/vlib/v/builder/prefs.v
@@ -0,0 +1,30 @@
+module builder
+
+pub struct Preferences {
+pub mut:
+	// paths
+	vpath string
+	vlib_path string
+	compile_dir string // contains os.realpath() of the dir of the final file beeing compiled, or the dir itself when doing `v .`
+	mod_path string
+	user_mod_path string
+	module_search_paths []string
+	// settings
+	os OS // the OS to build for
+	is_test bool
+	is_verbose bool
+}
+
+pub enum OS {
+	mac
+	linux
+	windows
+	freebsd
+	openbsd
+	netbsd
+	dragonfly
+	js // TODO
+	android
+	solaris
+	haiku
+}
diff --git a/vlib/v/checker/checker.v b/vlib/v/checker/checker.v
index c640614baf..23ceb6eadd 100644
--- a/vlib/v/checker/checker.v
+++ b/vlib/v/checker/checker.v
@@ -28,9 +28,9 @@ pub fn (c &Checker) check(ast_file ast.File) {
 	}
 }
 
-pub fn (c mut Checker) check_files(v_files []string, ast_files []ast.File) {
-	for i, file in ast_files {
-		c.file_name = v_files[i]
+pub fn (c mut Checker) check_files(ast_files []ast.File) {
+	for file in ast_files {
+		c.file_name = file.path
 		c.check(file)
 	}
 }
diff --git a/vlib/v/gen/cgen_test.v b/vlib/v/gen/cgen_test.v
index ef781d753f..a15e6ee01a 100644
--- a/vlib/v/gen/cgen_test.v
+++ b/vlib/v/gen/cgen_test.v
@@ -23,7 +23,7 @@ fn test_c_files() {
 		ctext := os.read_file('$vroot/vlib/v/gen/tests/${i}.c') or {
 			panic(err)
 		}
-		mut b := builder.new_builder()
+		mut b := builder.new_builder(builder.Preferences{})
 		res := b.gen_c([path])
 		if compare_texts(res, ctext) {
 			eprintln('${term_ok} ${i}')
diff --git a/vlib/v/gen/tests/1.vv b/vlib/v/gen/tests/1.vv
index c4bb2db3ad..b0690384f9 100644
--- a/vlib/v/gen/tests/1.vv
+++ b/vlib/v/gen/tests/1.vv
@@ -1,5 +1,5 @@
-import moda
-import modb as mb
+//import moda
+//import modb as mb
 
 const (
 	pi = 3
diff --git a/vlib/v/parser/parser.v b/vlib/v/parser/parser.v
index 6e4541ef94..1f46225123 100644
--- a/vlib/v/parser/parser.v
+++ b/vlib/v/parser/parser.v
@@ -88,6 +88,7 @@ pub fn parse_file(path string, table &table.Table) ast.File {
 	// println('nr stmts = $stmts.len')
 	// println(stmts[0])
 	return ast.File{
+		path: path
 		mod: module_decl
 		imports: imports
 		stmts: stmts