tools,vls: add `v test-parser examples/hello_world.v` (#7145)

cmd/tools/vtest-parser.v

@@ -0,0 +1,216 @@
+import os
+import flag
+import term
+import time
+import v.parser
+import v.table
+import v.ast
+import v.pref
+
+const (
+	vexe          = pref.vexe_path()
+	vroot         = os.dir(vexe)
+	support_color = term.can_show_color_on_stderr() && term.can_show_color_on_stdout()
+)
+
+struct Context {
+mut:
+	is_help    bool
+	is_worker  bool
+	is_verbose bool
+	myself     string // path to this executable, so the supervisor can launch worker processes
+	all_paths  []string // all files given to the supervisor process
+	path       string // the current path, given to a worker process
+	cut_index  int // the cut position in the source from context.path
+	// parser context in the worker processes:
+	table      table.Table
+	scope      ast.Scope
+	pref       pref.Preferences
+}
+
+fn main() {
+	mut context := process_cli_args()
+	if context.is_worker {
+		pid := os.getpid()
+		context.log('> worker ${pid:5} starts parsing at cut_index: ${context.cut_index:5} | $context.path')
+		// A worker's process job is to try to parse a single given file in context.path.
+		// It can crash/panic freely.
+		context.table = table.new_table()
+		context.scope = &ast.Scope{
+			parent: 0
+		}
+		context.pref = &pref.Preferences{
+			output_mode: .silent
+		}
+		mut source := os.read_file(context.path) ?
+		source = source[..context.cut_index]
+		_ := parser.parse_text(source, context.path, context.table, .skip_comments, context.pref,
+			context.scope)
+		context.log('> worker ${pid:5} finished parsing $context.path')
+		exit(0)
+	} else {
+		// The process supervisor should NOT crash/panic, unlike the workers.
+		// It's job, is to:
+		// 1) start workers
+		// 2) accumulate results
+		// 3) produce a summary at the end
+		context.expand_all_paths()
+		mut fails := 0
+		mut panics := 0
+		for path in context.all_paths {
+			new_fails, new_panics := context.process_whole_file_in_worker(path)
+			fails += new_fails
+			panics += new_panics
+		}
+		non_panics := fails - panics
+		println('Files processed: ${context.all_paths.len:5} | Errors found: ${fails:5} | Panics: ${panics:5} | Non panics: ${non_panics:5}')
+		if fails > 0 {
+			exit(1)
+		}
+		exit(0)
+	}
+}
+
+fn process_cli_args() &Context {
+	mut context := &Context{}
+	context.myself = os.executable()
+	mut fp := flag.new_flag_parser(os.args_after('test-parser'))
+	fp.application(os.file_name(context.myself))
+	fp.version('0.0.1')
+	fp.description('Test the V parser, by parsing each .v file in each PATH,\n' +
+		'as if it was typed character by character by the user.\n' +
+		'A PATH can be either a folder, or a specific .v file.\n' +
+		'NB: you *have to quote* the PATH, if it contains spaces/punctuation.')
+	fp.arguments_description('PATH1 PATH2 ...')
+	fp.skip_executable()
+	context.is_help = fp.bool('help', `h`, false, 'Show help/usage screen.')
+	context.is_verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
+	context.is_worker = fp.bool('worker', `w`, false, 'worker specific flag - is this a worker process, that can crash/panic.')
+	context.cut_index = fp.int('cut_index', `c`, 1, 'worker specific flag - cut index in the source file, everything before that will be parsed, the rest - ignored.')
+	context.path = fp.string('path', `p`, '', 'worker specific flag - path to the current source file, which will be parsed.')
+	//
+	if context.is_help {
+		println(fp.usage())
+		exit(0)
+	}
+	context.all_paths = fp.finalize() or {
+		context.error(err)
+		exit(1)
+	}
+	if !context.is_worker && context.all_paths.len == 0 {
+		println(fp.usage())
+		exit(0)
+	}
+	return context
+}
+
+// ////////////////
+fn bold(msg string) string {
+	if !support_color {
+		return msg
+	}
+	return term.bold(msg)
+}
+
+fn red(msg string) string {
+	if !support_color {
+		return msg
+	}
+	return term.red(msg)
+}
+
+fn yellow(msg string) string {
+	if !support_color {
+		return msg
+	}
+	return term.yellow(msg)
+}
+
+fn italic(msg string) string {
+	if !support_color {
+		return msg
+	}
+	return term.italic(msg)
+}
+
+fn (mut context Context) log(msg string) {
+	if context.is_verbose {
+		label := yellow('info')
+		ts := time.now().format_ss_micro()
+		eprintln('$label: $ts | $msg')
+	}
+}
+
+fn (mut context Context) error(msg string) {
+	label := red('error')
+	eprintln('$label: $msg')
+}
+
+fn (mut context Context) expand_all_paths() {
+	context.log('> context.all_paths before: $context.all_paths')
+	mut files := []string{}
+	for path in context.all_paths {
+		if os.is_dir(path) {
+			files << os.walk_ext(path, '.v')
+			files << os.walk_ext(path, '.vsh')
+			continue
+		}
+		if !path.ends_with('.v') && !path.ends_with('.vv') && !path.ends_with('.vsh') {
+			context.error('`v test-parser` can only be used on .v/.vv/.vsh files.\nOffending file: "$path".')
+			continue
+		}
+		if !os.exists(path) {
+			context.error('"$path" does not exist.')
+			continue
+		}
+		files << path
+	}
+	context.all_paths = files
+	context.log('> context.all_paths after: $context.all_paths')
+}
+
+fn (mut context Context) process_whole_file_in_worker(path string) (int, int) {
+	context.log('> context.process_whole_file_in_worker path: $path')
+	if !(os.is_file(path) && os.is_readable(path)) {
+		context.error('$path is not readable')
+		return 1, 0
+	}
+	source := os.read_file(path) or { '' }
+	if source == '' {
+		// an empty file is a valid .v file
+		return 0, 0
+	}
+	len := source.len - 1
+	mut fails := 0
+	mut panics := 0
+	for i in 0 .. len {
+		verbosity := if context.is_verbose { '-v' } else { '' }
+		cmd := '"$context.myself" $verbosity --worker --path "$path" --cut_index $i'
+		context.log(cmd)
+		res := os.exec(cmd) or { os.Result{
+			output: err
+			exit_code: 123
+		} }
+		context.log('worker exit_code: $res.exit_code | worker output:\n$res.output')
+		if res.exit_code != 0 {
+			fails++
+			mut is_panic := false
+			if res.output.contains('V panic:') {
+				is_panic = true
+				panics++
+			}
+			part := source[..i]
+			line := part.count('\n') + 1
+			last_line := part.all_after_last('\n')
+			col := last_line.len
+			err := if is_panic { red('parser failure: panic') } else { red('parser failure: crash') }
+			path_to_line := bold('$path:$line:$col:')
+			err_line := italic(last_line.trim_left('\t'))
+			println('$path_to_line $err')
+			println('\t$line | $err_line')
+			println('')
+			eprintln(res.output)
+		}
+	}
+	return fails, panics
+}

cmd/v/help/other.txt

@@ -10,5 +10,9 @@ but which are used less frequently by users:
    test-fmt          Test if all files in the current directory are formatted properly.
    test-compiler     Test if V is working properly by running all tests, including the compiler ones.
                      NB: this can take a minute or two to run
+   test-parser       Test that the V parser works with the given files, as if
+                     they were typed by a human programmer, one character at a time.
+                     NB: *very slow* for longer files (tens of seconds for 1KB .v file).
+                     Mainly useful as a parser bug finder for the V Language Server project.
 
    setup-freetype    Setup thirdparty freetype on Windows.

cmd/v/v.v

@@ -20,6 +20,7 @@ const (
 		'bin2v',
 		'test',
 		'test-fmt',
+		'test-parser',
 		'test-compiler',
 		'test-fixed',
 		'test-cleancode',

vlib/os/args.v

@@ -0,0 +1,51 @@
+// Copyright (c) 2019-2020 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module os
+
+// args_after returns all os.args, located *after* a specified `cut_word`.
+// When `cut_word` is NOT found, os.args is returned unmodified.
+pub fn args_after(cut_word string) []string {
+	if args.len == 0 {
+		return []string{}
+	}
+	mut cargs := []string{}
+	if cut_word !in args {
+		cargs = args
+	} else {
+		mut found := false
+		cargs << args[0]
+		for a in args[1..] {
+			if a == cut_word {
+				found = true
+				continue
+			}
+			if !found {
+				continue
+			}
+			cargs << a
+		}
+	}
+	return cargs
+}
+
+// args_after returns all os.args, located *before* a specified `cut_word`.
+// When `cut_word` is NOT found, os.args is returned unmodified.
+pub fn args_before(cut_word string) []string {
+	if args.len == 0 {
+		return []string{}
+	}
+	mut cargs := []string{}
+	if cut_word !in args {
+		cargs = args
+	} else {
+		cargs << args[0]
+		for a in args[1..] {
+			if a == cut_word {
+				break
+			}
+			cargs << a
+		}
+	}
+	return cargs
+}

vlib/v/parser/v_parser_test.v

@@ -184,4 +184,5 @@ table := &table.Table{}
 	println('source: $s')
 	println('parsed: $x')
 	println('===================')
+}
 */