From a8a81a1708ae35a5cd1c330d60de63424bfedc59 Mon Sep 17 00:00:00 2001
From: Delyan Angelov <delian66@gmail.com>
Date: Mon, 4 Jan 2021 13:36:58 +0200
Subject: [PATCH] ci: add "Verify `v test` works" tasks

---
 .github/workflows/ci.yml                | 32 +++++++++++
 cmd/tools/test_if_v_test_system_works.v | 74 +++++++++++++++++++++++++
 2 files changed, 106 insertions(+)
 create mode 100644 cmd/tools/test_if_v_test_system_works.v

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 254b3a5a1c..1d230b2f47 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -100,6 +100,10 @@ jobs:
     - name: v doctor
       run: |
         ./v doctor
+    - name: Verify `v test` works
+      run: |
+        ./v cmd/tools/test_if_v_test_system_works.v
+        ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: ./v -silent test-fixed
     - name: Test time functions in a timezone UTC-12
@@ -190,6 +194,10 @@ jobs:
 #      run: ./v build-vbinaries
 ##    - name: Test v->js
 ##      run: ./v -o hi.js examples/hello_v_js.v && node hi.js
+    - name: Verify `v test` works
+      run: |
+        ./v cmd/tools/test_if_v_test_system_works.v
+        ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: VJOBS=1 ./v -silent test-fixed
     - name: Build examples
@@ -267,6 +275,10 @@ jobs:
       run: |
         ./v -o v2 -usecache cmd/v
         ./v -usecache examples/tetris/tetris.v
+    - name: Verify `v test` works
+      run: |
+        ./v cmd/tools/test_if_v_test_system_works.v
+        ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
         ./v -silent test-fixed
@@ -348,6 +360,10 @@ jobs:
       run: ./v -o v2 cmd/v && ./v2 -o v3 cmd/v && ./v3 -o v4 cmd/v
     - name: v self with -usecache
       run: ./v -o v2 -usecache cmd/v
+    - name: Verify `v test` works
+      run: |
+        ./v cmd/tools/test_if_v_test_system_works.v
+        ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
         ./v -silent test-fixed
@@ -488,6 +504,10 @@ jobs:
       - name: Build V
         run: |
           echo $VFLAGS && make -j4 && ./v -cg -o v cmd/v
+      - name: Verify `v test` works
+        run: |
+          ./v cmd/tools/test_if_v_test_system_works.v
+          ./cmd/tools/test_if_v_test_system_works
       - name: Test V fixed tests
         run: |
           ./v -silent test-fixed
@@ -549,6 +569,10 @@ jobs:
         .\v.exe setup-freetype
         .\.github\workflows\windows-install-sqlite.bat
         ## .\.github\workflows\windows-install-sdl.bat
+    - name: Verify `v test` works
+      run: |
+        ./v cmd/tools/test_if_v_test_system_works.v
+        ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
         .\v.exe -silent test-fixed
@@ -587,6 +611,10 @@ jobs:
     - name: v doctor
       run: |
         ./v doctor
+    - name: Verify `v test` works
+      run: |
+        ./v cmd/tools/test_if_v_test_system_works.v
+        ./cmd/tools/test_if_v_test_system_works
     - name: Fixed tests
       run: |
         ./v -cg cmd\tools\vtest-fixed.v
@@ -619,6 +647,10 @@ jobs:
         .\v.exe setup-freetype
         .\.github\workflows\windows-install-sqlite.bat
         ## .\.github\workflows\windows-install-sdl.bat
+    - name: Verify `v test` works
+      run: |
+        .\v.exe cmd/tools/test_if_v_test_system_works.v
+        .\cmd\tools\test_if_v_test_system_works.exe
     - name: Fixed tests
       run: |
         .\v.exe -silent test-fixed
diff --git a/cmd/tools/test_if_v_test_system_works.v b/cmd/tools/test_if_v_test_system_works.v
new file mode 100644
index 0000000000..fea28b4bb2
--- /dev/null
+++ b/cmd/tools/test_if_v_test_system_works.v
@@ -0,0 +1,74 @@
+module main
+
+// This program verifies that `v test` propagates errors
+// and that it exits with code 1, when at least 1 FAIL happen.
+import os
+import rand
+
+const (
+	vexe  = get_vexe_path()
+	vroot = os.dir(vexe)
+	tdir  = new_tdir()
+)
+
+fn get_vexe_path() string {
+	env_vexe := os.getenv('VEXE')
+	if env_vexe != '' {
+		return env_vexe
+	}
+	me := os.executable()
+	eprintln('me: $me')
+	mut vexe := os.join_path(os.dir(os.dir(os.dir(me))), 'v')
+	if os.user_os() == 'windows' {
+		vexe += '.exe'
+	}
+	return vexe
+}
+
+fn new_tdir() string {
+	tdir := os.join_path(os.temp_dir(), rand.ulid())
+	if os.exists(tdir) {
+		os.rmdir(tdir)
+	}
+	os.mkdir(tdir)
+	C.atexit(cleanup_tdir)
+	return tdir
+}
+
+fn cleanup_tdir() {
+	println('... removing tdir: $tdir')
+	os.rmdir_all(tdir)
+}
+
+fn main() {
+	println('> vroot: $vroot | vexe: $vexe | tdir: $tdir')
+	ok_fpath := os.join_path(tdir, 'single_test.v')
+	os.write_file(ok_fpath, 'fn test_ok(){ assert true }')
+	check_ok('"$vexe" $ok_fpath')
+	check_ok('"$vexe" test $ok_fpath')
+	fail_fpath := os.join_path(tdir, 'failing_test.v')
+	os.write_file(fail_fpath, 'fn test_fail(){ assert 1 == 2 }')
+	check_fail('"$vexe" $fail_fpath')
+	check_fail('"$vexe" test $fail_fpath')
+	check_fail('"$vexe" test $tdir')
+}
+
+fn check_ok(cmd string) string {
+	println('>   check_ok cmd: $cmd')
+	res := os.exec(cmd) or { panic(err) }
+	if res.exit_code != 0 {
+		eprintln('>   check_ok failed.\n$res.output')
+		exit(1)
+	}
+	return res.output
+}
+
+fn check_fail(cmd string) string {
+	println('> check_fail cmd: $cmd')
+	res := os.exec(cmd) or { panic(err) }
+	if res.exit_code == 0 {
+		eprintln('> check_fail succeeded, but it should have failed.\n$res.output')
+		exit(1)
+	}
+	return res.output
+}