tests: add `v test-all`, move `v test-fixed` to `v test-compiler-full`

.github/PULL_REQUEST_TEMPLATE

@@ -23,13 +23,7 @@ fn test_foo() {
 
 If you are fixing a bug, please add a test that covers it.
 
-Before submitting a PR, please:
+Before submitting a PR, please run `v test-all` .
-  A) run the tests with `v test-compiler` .
-  B) make sure, that V can still compile itself:
-```shell
-./v -o v cmd/v
-./v -o v cmd/v
-```
 See also `TESTS.md`.
 
 I try to process PRs as soon as possible. They should be handled within 24 hours.

.github/workflows/alpine.test.sh

@@ -10,12 +10,12 @@ du -s .
 
 ls -lat 
 
-##./v test-compiler
+##./v test-all
 
 ## try running the known failing tests first to get faster feedback
 ./v test vlib/builtin/string_test.v vlib/strings/builder_test.v
 
-./v test-fixed
+./v test-compiler-full
 
 ./v build-vbinaries
 

.github/workflows/ci.yml

@@ -94,7 +94,7 @@ jobs:
       run: |
         thirdparty/tcc/tcc.exe -version
         ./v -cg -o v cmd/v # Make sure vtcc can build itself twice
-#        ./v -silent test-compiler
+#        ./v -silent test-all
     - name: v self compilation
       run: ./v -o v2 cmd/v && ./v2 -o v3 cmd/v && ./v3 -o v4 cmd/v
     - name: v doctor
@@ -105,7 +105,7 @@ jobs:
         ./v cmd/tools/test_if_v_test_system_works.v
         ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
-      run: ./v -silent test-fixed
+      run: ./v -silent test-compiler-full
     - name: Test time functions in a timezone UTC-12
       run: TZ=Etc/GMT+12 ./v test vlib/time/
     - name: Test time functions in a timezone UTC-3
@@ -155,7 +155,7 @@ jobs:
           env
       - name: Test V fixed tests
         run: |
-          ./v -silent test-fixed
+          ./v -silent test-compiler-full
 
   macos:
     runs-on: macOS-latest
@@ -195,7 +195,7 @@ jobs:
 #        psql -d postgres -c 'create database customerdb;'
 #        psql -d customerdb -f examples/database/pg/mydb.sql
 #    - name: Test v->c
-#      run: ./v -silent test-compiler
+#      run: ./v -silent test-all
 #    - name: Test v binaries
 #      run: ./v build-vbinaries
 ##    - name: Test v->js
@@ -205,7 +205,7 @@ jobs:
         ./v cmd/tools/test_if_v_test_system_works.v
         ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
-      run: VJOBS=1 ./v -silent test-fixed
+      run: VJOBS=1 ./v -silent test-compiler-full
     - name: Build examples
       run: ./v build-examples
     - name: Build examples with -autofree
@@ -264,7 +264,7 @@ jobs:
         UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 ./v2 -o v.c cmd/v
         UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 ./v3 -o v.c cmd/v
 #    - name: Test V
-#      run: ./v -silent test-compiler
+#      run: ./v -silent test-all
 #    - name: Test v binaries
 #      run: ./v build-vbinaries
 ##    - name: Test v->js
@@ -287,9 +287,9 @@ jobs:
         ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
-        ./v -silent test-fixed
+        ./v -silent test-compiler-full
     - name: Fixed tests (-prod)
-      run: ./v -o vprod -prod cmd/v && ./vprod -silent test-fixed
+      run: ./v -o vprod -prod cmd/v && ./vprod -silent test-compiler-full
     - name: Build examples
       run: ./v build-examples
     - name: Build examples with -autofree
@@ -372,9 +372,9 @@ jobs:
         ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
-        ./v -silent test-fixed
+        ./v -silent test-compiler-full
     - name: Fixed tests (-prod)
-      run: ./v -o vprod -prod cmd/v && ./vprod -silent test-fixed
+      run: ./v -o vprod -prod cmd/v && ./vprod -silent test-compiler-full
     - name: Build examples
       run: ./v build-examples
     - name: Build examples with -autofree
@@ -429,7 +429,7 @@ jobs:
     - name: Build V
       run: make -j4 && ./v -cc clang -cg -cflags -Werror -o v cmd/v
     - name: Fixed tests (-fsanitize=undefined)
-      run: ./v -cc clang -cflags "-fsanitize=undefined" -o v2 cmd/v && ./v2 -cflags -fsanitize=undefined test-fixed
+      run: ./v -cc clang -cflags "-fsanitize=undefined" -o v2 cmd/v && ./v2 -cflags -fsanitize=undefined test-compiler-full
 
   tests-sanitize-address:
     runs-on: ubuntu-20.04
@@ -452,7 +452,7 @@ jobs:
     - name: Build V
       run: make -j4 && ./v -cc clang -cg -cflags -Werror -o v cmd/v
     - name: Fixed tests (-fsanitize=address)
-      run: ASAN_OPTIONS=detect_leaks=0 ./v -cflags -fsanitize=address test-fixed
+      run: ASAN_OPTIONS=detect_leaks=0 ./v -cflags -fsanitize=address test-compiler-full
 
   tests-sanitize-memory:
     runs-on: ubuntu-20.04
@@ -475,7 +475,7 @@ jobs:
     - name: Build V
       run: make -j4 && ./v -cc clang -cg -cflags -Werror -o v cmd/v
     - name: Fixed tests (-fsanitize=memory)
-      run: ./v -cflags -fsanitize=memory test-fixed
+      run: ./v -cflags -fsanitize=memory test-compiler-full
 
 #  ubuntu-autofree-selfcompile:
 #    runs-on: ubuntu-20.04
@@ -516,7 +516,7 @@ jobs:
           ./cmd/tools/test_if_v_test_system_works
       - name: Test V fixed tests
         run: |
-          ./v -silent test-fixed
+          ./v -silent test-compiler-full
 
 #   ubuntu-musl:
 #     runs-on: ubuntu-20.04
@@ -544,7 +544,7 @@ jobs:
 #     - name: quick debug
 #       run: ./v -stats vlib/strconv/format_test.v
 #     - name: Fixed tests
-#       run: ./v -silent test-fixed
+#       run: ./v -silent test-compiler-full
 
   ubuntu-llvm-mingw:
     runs-on: ubuntu-20.04
@@ -581,10 +581,10 @@ jobs:
         ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
-        .\v.exe -silent test-fixed
+        .\v.exe -silent test-compiler-full
 #    - name: Test
 #      run: |
-#        .\v.exe -silent test-compiler
+#        .\v.exe -silent test-all
       ## v.js dosent work on windows
         #.\v.exe -o hi.js examples/hello_v_js.v
         #node hi.js
@@ -623,13 +623,13 @@ jobs:
         ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
-        ./v -cg cmd\tools\vtest-fixed.v
+        ./v -cg cmd\tools\vtest-compiler-full.v
-        ./v -silent test-fixed
+        ./v -silent test-compiler-full
     - name: Build examples
       run: ./v build-examples
 #    - name: Test
 #      run: |
-#        .\v.exe -silent test-compiler
+#        .\v.exe -silent test-all
 #      ## v.js dosent work on windows
         #.\v.exe -o hi.js examples/hello_v_js.v
         #node hi.js
@@ -659,10 +659,10 @@ jobs:
         .\cmd\tools\test_if_v_test_system_works.exe
     - name: Fixed tests
       run: |
-        .\v.exe -silent test-fixed
+        .\v.exe -silent test-compiler-full
 #    - name: Test
 #      run: |
-#        .\v.exe -silent test-compiler
+#        .\v.exe -silent test-all
       ## v.js dosent work on windows
         #.\v.exe -o hi.js examples/hello_v_js.v
         #node hi.js
@@ -728,7 +728,7 @@ jobs:
     - name: V self compilation with g++
       run: ./v -cc g++-9 -o v2 cmd/v && ./v2 -cc g++-9 -o v3 cmd/v
 ##    - name: Running tests with g++
-##      run: ./v -cc g++-9 -silent test-fixed
+##      run: ./v -cc g++-9 -silent test-compiler-full
 
   install-modules:
     runs-on: ubuntu-20.04

.github/workflows/periodic.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Symlink V
       run:  sudo ./v symlink
     - name: Run network tests
-      run: ./v -d network -silent test-fixed
+      run: ./v -d network -silent test-compiler-full
 
 
   network-tests-macos:
@@ -38,7 +38,7 @@ jobs:
     - name: Ensure thirdparty/cJSON/cJSON.o is compiled, before running tests.
       run:  ./v examples/json.v
     - name: Run network tests
-      run: ./v -d network -silent test-fixed
+      run: ./v -d network -silent test-compiler-full
 
 
   network-windows-msvc:
@@ -55,4 +55,4 @@ jobs:
         echo $VFLAGS
         .\make.bat -msvc
     - name: Run network tests
-      run: .\v.exe -d network -silent test-fixed
+      run: .\v.exe -d network -silent test-compiler-full

CONTRIBUTING.md

@@ -143,7 +143,7 @@ feature/bugfix* that you make.
 
 ### Testing your commits locally:
 You can test locally whether your changes have not broken something by
-running: `v test-compiler`. See `TESTS.md` for more details.
+running: `v test-all`. See `TESTS.md` for more details.
 
 ### Publishing your commits to GitHub:
 

TESTS.md

@@ -1,5 +1,9 @@
 # Automated tests
 
+TLDR: run `v test-all` locally, after making your changes, 
+and before submitting PRs.
+
+## Notes
 In the `v` repo there are several different tests. The main types are:
 
 * `_test.v` tests - check that `test_` functions succeed. These can be 
@@ -10,61 +14,74 @@ particularly useful for checking that errors are printed.
 
 Tip: use `v -cc tcc` when compiling tests for speed.
 
-
-## `v test-compiler`
-
-This builds and tests:
-
-* `vc` repo
-* `_test.v` files
-* examples
-* `v install` a module
-
-Some of these can be invoked separately.
-
-## `vlib/v/compiler_errors_test.v`
-
-This runs tests for:
-* `checker/tests/*.vv`
-* `parser/tests/*.vv`
-
 ## `vlib/v/tests`
 
-General runnable program tests.
+General runnable tests for different features of the V compiler.
 
-## Actual code
+## Test building of actual V programs (examples, tools, V itself)
 
-* `v build-examples`
 * `v build-tools`
-
+* `v build-examples`
-## `v test-fixed`
+* `v build-vbinaries`
-
-Test `vlib`.
 
 ## vfmt tests
 
-In `vlib/v/fmt/` there's:
+In `vlib/v/fmt/` there are::
 
-* `fmt_test.v`
+* `v vlib/v/fmt/fmt_test.v`
 
 This checks `.out` tests.
 
-* `fmt_keep_test.v`
+* `v vlib/v/fmt/fmt_keep_test.v`
 
 This verifies that `_keep.v` files would be unchanged by `vfmt -w`.
 
-* `fmt_vlib_test.v`
+* `v vlib/v/fmt/fmt_vlib_test.v`
 
-This checks all source files are formatted and prints a summary. This 
+This checks all source files are formatted and prints a summary.
-is not required.
+This is not required.
 
+## Other
 * `v test-fmt`
 
 Test all files in the current directory are formatted.
 
+* `v run cmd/tools/check-md.v -hide-warnings -all`
+
+Ensure that all .md files in the project are formatted properly,
+and that the V code block examples in them can be compiled/formatted too.
+
 ## `.github/workflows/ci.yml`
 
 This runs various CI tests, e.g.:
 
 * `v vet vlib/v` - style checker
 * `v fmt -verify` on certain source files
+
+## `v test-cleancode`
+
+Check that most .v files, are invariant of `v fmt` runs.
+
+## `v test-compiler-full`
+
+Run `vlib` module tests, *including* the compiler tests.
+
+## `v vlib/v/compiler_errors_test.v`
+
+This runs tests for:
+* `checker/tests/*.vv`
+* `parser/tests/*.vv`
+
+## `v test-all`
+
+Test and build *everything*. Usefull to verify *locally*, that the CI will
+most likely pass. Slowest, but most comprehensive. 
+
+It works, by running these in succession:
+* `v test-cleancode`
+* `v test-compiler-full`
+* `v test-fmt`
+* `v build-tools`
+* `v build-examples`
+* `v run cmd/tools/check-md.v -hide-warnings -all`
+* `v install nedpals.args`

cmd/tools/check_os_api_parity.v

@@ -40,9 +40,7 @@ fn main() {
 	vroot := os.dir(vexe)
 	util.set_vroot_folder(vroot)
 	os.chdir(vroot)
-	cmd := util.find_working_diff_command() or {
+	cmd := util.find_working_diff_command() or { '' }
-		''
-	}
 	mut app := App{
 		diff_cmd: cmd
 		is_verbose: os.getenv('VERBOSE').len > 0
@@ -63,7 +61,7 @@ fn main() {
 	}
 	howmany := app.api_differences.len
 	eprintln('NB: please, do run `git clean -xf` after this tool, or at least `find thirdparty/ |grep .o$|xargs rm`')
-	eprintln('otherwise, `./v test-fixed` may show false positives, due to .o files compiled with a cross compiler.')
+	eprintln('otherwise, `./v test-compiler-full` may show false positives, due to .o files compiled with a cross compiler.')
 	if howmany > 0 {
 		eprintln(term.header('Found $howmany modules with different APIs', '='))
 		for m in app.api_differences.keys() {
@@ -105,7 +103,7 @@ fn (app App) gen_api_for_module_in_os(mod_name string, os_name string) string {
 		for s in f.stmts {
 			if s is ast.FnDecl {
 				if s.is_pub {
-					fn_signature := s.stringify(b.table, mod_name, map[string]string)
+					fn_signature := s.stringify(b.table, mod_name, map[string]string{})
 					fn_mod := s.modname()
 					if fn_mod == mod_name {
 						fline := '$fn_mod: $fn_signature'

cmd/tools/vcomplete.v

@@ -53,8 +53,7 @@ const (
 		'bin2v',
 		'test',
 		'test-fmt',
-		'test-compiler',
+		'test-compiler-full',
-		'test-fixed',
 		'test-cleancode',
 		'repl',
 		'complete',
@@ -283,7 +282,7 @@ compdef _v v
 			mut lines := []string{}
 			list := auto_complete_request(sub_args[1..])
 			for entry in list {
-				lines << "compadd -U -S \"\" -- '$entry';"
+				lines << 'compadd -U -S' + '""' + ' -- ' + "'$entry';"
 			}
 			println(lines.join('\n'))
 		}

cmd/tools/vtest-all.v

@@ -0,0 +1,106 @@
+module main
+
+import os
+import term
+import time
+
+const vexe = os.getenv('VEXE')
+
+const vroot = os.dir(vexe)
+
+const args_string = os.args[1..].join(' ')
+
+const vargs = args_string.all_before('test-all')
+
+fn main() {
+	commands := get_all_commands()
+	commands.summary()
+}
+
+struct Command {
+mut:
+	line   string
+	label  string // when set, the label will be printed *before* cmd.line is executed
+	ecode  int
+	okmsg  string
+	errmsg string
+}
+
+fn get_all_commands() []Command {
+	mut res := []Command{}
+	res << Command{
+		line: '$vexe $vargs -progress test-cleancode'
+		okmsg: 'All important .v files are invariant when processed with `v fmt`'
+	}
+	res << Command{
+		line: '$vexe $vargs -progress test-fmt'
+		okmsg: 'All .v files can be processed with `v fmt`. NB: the result may not always be compilable, it just means that `v fmt` does not crash.'
+	}
+	res << Command{
+		line: '$vexe $vargs -progress test-compiler-full'
+		okmsg: 'There are no _test.v file regressions.'
+	}
+	res << Command{
+		line: '$vexe $vargs -progress build-tools'
+		okmsg: 'All tools can be compiled.'
+	}
+	res << Command{
+		line: '$vexe $vargs -progress build-examples'
+		okmsg: 'All examples can be compiled.'
+	}
+	res << Command{
+		line: '$vexe run cmd/tools/check-md.v -hide-warnings -all'
+		label: 'Check ```v ``` code examples and formatting of .MD files...'
+		okmsg: 'All .md files look good.'
+	}
+	res << Command{
+		line: '$vexe install nedpals.args'
+		okmsg: '`v install` works.'
+	}
+	$if macos {
+		res << Command{
+			line: '$vexe -o v.c cmd/v && cc -Werror v.c && rm -rf v.c'
+			label: 'v.c should be buildable with no warnings...'
+			okmsg: 'v.c can be compiled without warnings. This is good :)'
+		}
+	}
+	return res
+}
+
+fn (mut cmd Command) run() {
+	// Changing the current directory is needed for some of the compiler tests,
+	// vlib/v/tests/local_test.v and vlib/v/tests/repl/repl_test.v
+	os.chdir(vroot)
+	if cmd.label != '' {
+		println(term.header(cmd.label, '*'))
+	}
+	sw := time.new_stopwatch({})
+	cmd.ecode = os.system(cmd.line)
+	spent := sw.elapsed().milliseconds()
+	println(term.yellow('> Running: "$cmd.line" took: $spent ms.'))
+	println('')
+}
+
+fn (commands []Command) summary() {
+	sw := time.new_stopwatch({})
+	for mut cmd in commands {
+		cmd.run()
+	}
+	spent := sw.elapsed().milliseconds()
+	oks := commands.filter(it.ecode == 0)
+	fails := commands.filter(it.ecode != 0)
+	println('')
+	println(term.header(term.yellow(term.bold('Summary of `v test-all`:')), '-'))
+	println(term.yellow('Total runtime: $spent ms'))
+	for ocmd in oks {
+		msg := if ocmd.okmsg != '' { ocmd.okmsg } else { ocmd.line }
+		println(term.green('>          OK: $msg '))
+	}
+	for fcmd in fails {
+		msg := if fcmd.errmsg != '' { fcmd.errmsg } else { fcmd.line }
+		println(term.red('>      Failed: $msg '))
+	}
+	if fails.len > 0 {
+		exit(1)
+	}
+}

cmd/tools/vtest-compiler-full.v

@@ -95,7 +95,7 @@ fn main() {
 	os.chdir(vroot)
 	args := os.args.clone()
 	args_string := args[1..].join(' ')
-	cmd_prefix := args_string.all_before('test-fixed')
+	cmd_prefix := args_string.all_before('test-compiler-full')
 	title := 'testing all fixed tests'
 	all_test_files := os.walk_ext(os.join_path(vroot, 'vlib'), '_test.v')
 	testing.eheader(title)

cmd/tools/vtest-compiler.v

@@ -1,70 +0,0 @@
-module main
-
-import os
-import testing
-import benchmark
-import v.pref
-
-fn main() {
-	args_string := os.args[1..].join(' ')
-	v_test_compiler(args_string.all_before('test-compiler'))
-}
-
-fn v_test_compiler(vargs string) {
-	vexe := pref.vexe_path()
-	parent_dir := os.dir(vexe)
-	testing.vlib_should_be_present(parent_dir)
-	// Changing the current directory is needed for some of the compiler tests,
-	// vlib/v/tests/local_test.v and vlib/v/tests/repl/repl_test.v
-	os.chdir(parent_dir)
-	/*
-	if !os.exists(parent_dir + '/v.v') {
-		eprintln('v.v is missing, it must be next to the V executable')
-		exit(1)
-	}
-	*/
-	// Make sure v.c can be compiled without warnings
-	$if macos {
-		if os.exists('/cmd/v') {
-			os.system('$vexe -o v.c cmd/v')
-			if os.system('cc -Werror v.c') != 0 {
-				eprintln('cc failed to build v.c without warnings')
-				exit(1)
-			}
-			eprintln('v.c can be compiled without warnings. This is good :)')
-		}
-	}
-	building_tools_failed := os.system('"$vexe" build-tools') != 0
-	eprintln('')
-	testing.eheader('Testing all _test.v files...')
-	mut compiler_test_session := testing.new_test_session(vargs)
-	compiler_test_session.files << os.walk_ext(parent_dir, '_test.v')
-	compiler_test_session.test()
-	eprintln(compiler_test_session.benchmark.total_message('running V tests'))
-	eprintln('')
-	building_examples_failed := testing.v_build_failing(vargs, 'examples')
-	eprintln('')
-	building_live_failed := testing.v_build_failing(vargs + '-live', os.join_path('examples',
-		'hot_reload'))
-	eprintln('')
-	//
-	testing.setup_new_vtmp_folder()
-	v_module_install_cmd := '$vexe install nedpals.args'
-	eprintln('')
-	testing.eheader('Installing a v module with: $v_module_install_cmd')
-	mut vmark := benchmark.new_benchmark()
-	ret := os.system(v_module_install_cmd)
-	if ret != 0 {
-		eprintln('failed to run v install')
-	}
-	desired_path := os.join_path(pref.default_module_path, 'nedpals', 'args')
-	if !(os.exists(desired_path) && os.is_dir(desired_path)) {
-		eprintln('v failed to install a test module')
-	}
-	vmark.stop()
-	eprintln('Installing a v module took: ' + vmark.total_duration().str() + 'ms')
-	//
-	if building_tools_failed || compiler_test_session.failed || building_examples_failed || building_live_failed {
-		exit(1)
-	}
-}

cmd/tools/vtest.v

@@ -15,7 +15,7 @@ fn main() {
 	args_after := cmdline.options_after(args_to_executable, ['test'])
 	if args_after.join(' ') == 'v' {
 		eprintln('`v test v` has been deprecated.')
-		eprintln('Use `v test-compiler` instead.')
+		eprintln('Use `v test-all` instead.')
 		exit(1)
 	}
 	mut ts := testing.new_test_session(args_before.join(' '))

cmd/v/help/other.txt

@@ -7,9 +7,16 @@ but which are used less frequently by users:
    build-examples     Test if all examples can be built.
    build-tools        Test if all tools can be built.
    build-vbinaries    Test if V can be built with different configuration.
+
+   test-all           Run most checks, that the CI does locally.
+                      It may take over 2 minutes, and it needs internet connectivity too,
+                      because it tries to also verify that `v install` works.
+
    test-fmt           Test if all files in the current directory are formatted properly.
-   test-compiler     Test if V is working properly by running all tests, including the compiler ones.
+
-                     NB: this can take a minute or two to run
+   test-compiler-full Test if V is working properly by running all tests, including the compiler ones.
+                      NB: this can 1-2 minutes to run.
+
    test-parser        Test that the V parser works with the given files, as if
                       they were typed by a human programmer, one character at a time.
                       NB: *very slow* for longer files (tens of seconds for 1KB .v file).
@@ -18,4 +25,3 @@ but which are used less frequently by users:
    setup-freetype     Setup thirdparty freetype on Windows.
 
    wipe-cache         Remove the V cache folder. Useful for cleaning the cache, and guaranteeing a clean build.
-   

cmd/v/v.v

@@ -19,10 +19,12 @@ const (
 		'symlink',
 		'bin2v',
 		'test',
+		'test-all', /* runs most of the tests and other checking tools, that will be run by the CI */
 		'test-fmt',
 		'test-parser',
-		'test-compiler',
+		'test-fixed', /* deprecated by test-compiler-full */
-		'test-fixed',
+		'test-compiler', /* deprecated by test-compiler-full */
+		'test-compiler-full',
 		'test-cleancode',
 		'repl',
 		'complete',
@@ -77,6 +79,14 @@ fn main() {
 		eprintln('-usecache is currently disabled on windows')
 		exit(1)
 	}
+	if command == 'test-fixed' {
+		eprintln('Please use `v test-compiler-full` instead.')
+		exit(1)
+	}
+	if command == 'test-compiler' {
+		eprintln('Please use either `v test-all`, `v test-compiler-full`, `v build-examples`, `v build-tools` or `v build-vbinaries` instead.')
+		exit(1)
+	}
 	if command == 'test-vet' {
 		println('Please use `v test-cleancode` instead.')
 		return