all: support compile time `$env('ENV_VAR')` (#8456)

CHANGELOG.md

@@ -26,6 +26,7 @@
 from local variables.
 - `__offsetof` for low level needs (works like `offsetof` in C).
 - vfmt now preserves empty lines, like gofmt.
+- Support for compile time environment variables via `$env('ENV_VAR')`. 
 
 ## V 0.2.1
 *30 Dec 2020*

cmd/tools/vtest.v

@@ -35,7 +35,7 @@ fn main() {
 	}
 	testing.header('Testing...')
 	ts.test()
-	println(ts.benchmark.total_message('Ran all V _test.v files'))
+	println(ts.benchmark.total_message('all V _test.v files'))
 	if ts.failed {
 		exit(1)
 	}

doc/docs.md

@@ -3346,6 +3346,21 @@ executable, increasing your binary size, but making it more self contained
 and thus easier to distribute. In this case, `f.data()` will cause *no IO*,
 and it will always return the same data.
 
+#### $env
+
+```v
+module main
+
+fn main() {
+	compile_time_env := $env('ENV_VAR')
+	println(compile_time_env)
+}
+```
+
+V can bring in values at compile time from environment variables.
+`$env('ENV_VAR')` can also be used in top-level `#flag` and `#include` statements:
+`#flag linux -I $env('JAVA_HOME')/include`.
+
 ### Environment specific files
 
 If a file has an environment-specific suffix, it will only be compiled for that environment.

vlib/benchmark/benchmark.v

@@ -189,7 +189,8 @@ pub fn (b &Benchmark) step_message_skip(msg string) string {
 
 // total_message returns a string with total summary of the benchmark run.
 pub fn (b &Benchmark) total_message(msg string) string {
-	mut tmsg := '${term.colorize(term.bold, 'Summary:')} '
+	the_label := term.colorize(term.gray, msg)
+	mut tmsg := '${term.colorize(term.bold, 'Summary for $the_label:')} '
 	if b.nfail > 0 {
 		tmsg += term.colorize(term.bold, term.colorize(term.red, '$b.nfail failed')) + ', '
 	}
@@ -200,7 +201,6 @@ pub fn (b &Benchmark) total_message(msg string) string {
 		tmsg += term.colorize(term.bold, term.colorize(term.yellow, '$b.nskip skipped')) + ', '
 	}
 	tmsg += '$b.ntotal total. ${term.colorize(term.bold, 'Runtime:')} ${b.bench_timer.elapsed().microseconds() / 1000} ms.\n'
-	tmsg += term.colorize(term.gray, msg)
 	return tmsg
 }
 

vlib/v/ast/ast.v

@@ -1133,14 +1133,20 @@ pub:
 	method_pos  token.Position
 	scope       &Scope
 	left        Expr
+	args_var    string
+	//
 	is_vweb   bool
 	vweb_tmpl File
-	args_var    string
+	//
 	is_embed   bool
 	embed_file EmbeddedFile
+	//
+	is_env  bool
+	env_pos token.Position
 pub mut:
 	sym         table.TypeSymbol
 	result_type table.Type
+	env_value   string
 }
 
 pub struct None {

vlib/v/checker/checker.v

@@ -3204,6 +3204,14 @@ fn (mut c Checker) hash_stmt(mut node ast.HashStmt) {
 			}
 			node.val = 'include $vroot'
 			node.main = vroot
+			flag = vroot
+		}
+		if flag.contains('\$env(') {
+			env := util.resolve_env_value(flag, true) or {
+				c.error(err, node.pos)
+				return
+			}
+			node.main = env
 		}
 		flag_no_comment := flag.all_before('//').trim_space()
 		if !((flag_no_comment.starts_with('"') && flag_no_comment.ends_with('"'))
@@ -3239,6 +3247,12 @@ fn (mut c Checker) hash_stmt(mut node ast.HashStmt) {
 				return
 			}
 		}
+		if flag.contains('\$env(') {
+			flag = util.resolve_env_value(flag, true) or {
+				c.error(err, node.pos)
+				return
+			}
+		}
 		for deprecated in ['@VMOD', '@VMODULE', '@VPATH', '@VLIB_PATH'] {
 			if flag.contains(deprecated) {
 				c.error('$deprecated had been deprecated, use @VROOT instead.', node.pos)
@@ -3663,6 +3677,14 @@ pub fn (mut c Checker) cast_expr(mut node ast.CastExpr) table.Type {
 
 fn (mut c Checker) comptime_call(mut node ast.ComptimeCall) table.Type {
 	node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left)))
+	if node.is_env {
+		env_value := util.resolve_env_value("\$env('$node.args_var')", false) or {
+			c.error(err, node.env_pos)
+			return table.string_type
+		}
+		node.env_value = env_value
+		return table.string_type
+	}
 	if node.is_embed {
 		c.file.embedded_files << node.embed_file
 		return c.table.find_type_idx('v.embed_file.EmbedFileData')

vlib/v/checker/tests/comptime_env/env_parser_errors_1.run.out

@@ -0,0 +1,3 @@
+vlib/v/checker/tests/comptime_env/env_parser_errors_1.vv:1:3: error: supply an env variable name like HOME, PATH or USER
+    1 | #flag -I $env('')/xyz
+      |   ~~~~~~~~~~~~~~~~~~~

vlib/v/checker/tests/comptime_env/env_parser_errors_1.vv

@@ -0,0 +1 @@
+#flag -I $env('')/xyz

vlib/v/checker/tests/comptime_env/env_parser_errors_2.run.out

@@ -0,0 +1,3 @@
+vlib/v/checker/tests/comptime_env/env_parser_errors_2.vv:1:3: error: cannot use string interpolation in compile time $env() expression
+    1 | #flag -I $env('$ABC')/xyz
+      |   ~~~~~~~~~~~~~~~~~~~~~~~

vlib/v/checker/tests/comptime_env/env_parser_errors_2.vv

@@ -0,0 +1 @@
+#flag -I $env('$ABC')/xyz

vlib/v/checker/tests/comptime_env/env_parser_errors_3.run.out

@@ -0,0 +1,3 @@
+vlib/v/checker/tests/comptime_env/env_parser_errors_3.vv:1:3: error: no "$env('...')" could be found in "-I $env()/xyz".
+    1 | #flag -I $env()/xyz
+      |   ~~~~~~~~~~~~~~~~~

vlib/v/checker/tests/comptime_env/env_parser_errors_3.vv

@@ -0,0 +1 @@
+#flag -I $env()/xyz

vlib/v/checker/tests/comptime_env/using_comptime_env.run.out

@@ -0,0 +1,11 @@
+vlib/v/checker/tests/comptime_env/using_comptime_env.vv:1:3: error: the environment variable "VAR" does not exist.
+    1 | #flag -I $env('VAR')/xyz
+      |   ~~~~~~~~~~~~~~~~~~~~~~
+    2 | #include "$env('VAR')/stdio.h"
+    3 |
+vlib/v/checker/tests/comptime_env/using_comptime_env.vv:2:3: error: the environment variable "VAR" does not exist.
+    1 | #flag -I $env('VAR')/xyz
+    2 | #include "$env('VAR')/stdio.h"
+      |   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    3 | 
+    4 | fn main() {

vlib/v/checker/tests/comptime_env/using_comptime_env.var.run.out

@@ -0,0 +1,2 @@
+/usr/include
+done

vlib/v/checker/tests/comptime_env/using_comptime_env.var_invalid.run.out

@@ -0,0 +1 @@
+builder error: '/opt/invalid/path/stdio.h' not found

vlib/v/checker/tests/comptime_env/using_comptime_env.vv

@@ -0,0 +1,8 @@
+#flag -I $env('VAR')/xyz
+#include "$env('VAR')/stdio.h"
+
+fn main() {
+	env := $env('VAR')
+	println(env)
+	println('done')
+}

vlib/v/compiler_errors_test.v

@@ -19,6 +19,8 @@ const turn_off_vcolors = os.setenv('VCOLORS', 'never', true)
 
 const should_autofix = os.getenv('VAUTOFIX') != ''
 
+const github_job = os.getenv('GITHUB_JOB')
+
 struct TaskDescription {
 	vexe             string
 	dir              string
@@ -30,8 +32,19 @@ mut:
 	is_skipped        bool
 	is_module         bool
 	expected          string
+	expected_out_path string
 	found___          string
 	took              time.Duration
+	cli_cmd           string
+}
+
+struct Tasks {
+	vexe          string
+	parallel_jobs int // 0 is using VJOBS, anything else is an override
+	label         string
+mut:
+	show_cmd bool
+	all      []TaskDescription
 }
 
 fn test_all() {
@@ -52,28 +65,51 @@ fn test_all() {
 	module_tests := get_tests_in_dir(module_dir, true)
 	run_tests := get_tests_in_dir(run_dir, false)
 	// -prod is used for the parser and checker tests, so that warns are errors
-	mut tasks := []TaskDescription{}
+	mut tasks := Tasks{
-	tasks.add(vexe, parser_dir, '-prod', '.out', parser_tests, false)
+		vexe: vexe
-	tasks.add(vexe, checker_dir, '-prod', '.out', checker_tests, false)
+		label: 'all tests'
-	tasks.add(vexe, scanner_dir, '-prod', '.out', scanner_tests, false)
+	}
-	tasks.add(vexe, checker_dir, '-d mysymbol run', '.mysymbol.run.out', ['custom_comptime_define_error.vv'],
+	tasks.add('', parser_dir, '-prod', '.out', parser_tests, false)
+	tasks.add('', checker_dir, '-prod', '.out', checker_tests, false)
+	tasks.add('', scanner_dir, '-prod', '.out', scanner_tests, false)
+	tasks.add('', checker_dir, '-d mysymbol run', '.mysymbol.run.out', ['custom_comptime_define_error.vv'],
 		false)
-	tasks.add(vexe, checker_dir, '-d mydebug run', '.mydebug.run.out', ['custom_comptime_define_if_flag.vv'],
+	tasks.add('', checker_dir, '-d mydebug run', '.mydebug.run.out', ['custom_comptime_define_if_flag.vv'],
 		false)
-	tasks.add(vexe, checker_dir, '-d nodebug run', '.nodebug.run.out', ['custom_comptime_define_if_flag.vv'],
+	tasks.add('', checker_dir, '-d nodebug run', '.nodebug.run.out', ['custom_comptime_define_if_flag.vv'],
 		false)
-	tasks.add(vexe, checker_dir, '--enable-globals run', '.run.out', ['globals_error.vv'],
+	tasks.add('', checker_dir, '--enable-globals run', '.run.out', ['globals_error.vv'],
 		false)
-	tasks.add(vexe, global_dir, '--enable-globals', '.out', global_tests, false)
+	tasks.add('', global_dir, '--enable-globals', '.out', global_tests, false)
-	tasks.add(vexe, module_dir, '-prod run', '.out', module_tests, true)
+	tasks.add('', module_dir, '-prod run', '.out', module_tests, true)
-	tasks.add(vexe, run_dir, 'run', '.run.out', run_tests, false)
+	tasks.add('', run_dir, 'run', '.run.out', run_tests, false)
 	tasks.run()
+	if github_job == 'ubuntu-tcc' {
+		// these should be run serially, since they depend on setting and using environment variables
+		mut cte_tasks := Tasks{
+			vexe: vexe
+			parallel_jobs: 1
+			label: 'comptime env tests'
+		}
+		cte_dir := '$checker_dir/comptime_env'
+		files := get_tests_in_dir(cte_dir, false)
+		cte_tasks.add('', cte_dir, '-no-retry-compilation run', '.run.out', files, false)
+		cte_tasks.add('VAR=/usr/include $vexe', cte_dir, '-no-retry-compilation run',
+			'.var.run.out', ['using_comptime_env.vv'], false)
+		cte_tasks.add('VAR=/opt/invalid/path $vexe', cte_dir, '-no-retry-compilation run',
+			'.var_invalid.run.out', ['using_comptime_env.vv'], false)
+		cte_tasks.run()
+	}
 }
 
-fn (mut tasks []TaskDescription) add(vexe string, dir string, voptions string, result_extension string, tests []string, is_module bool) {
+fn (mut tasks Tasks) add(custom_vexe string, dir string, voptions string, result_extension string, tests []string, is_module bool) {
+	mut vexe := tasks.vexe
+	if custom_vexe != '' {
+		vexe = custom_vexe
+	}
 	paths := vtest.filter_vtest_only(tests, basepath: dir)
 	for path in paths {
-		tasks << TaskDescription{
+		tasks.all << TaskDescription{
 			vexe: vexe
 			dir: dir
 			voptions: voptions
@@ -89,12 +125,13 @@ fn bstep_message(mut bench benchmark.Benchmark, label string, msg string, sdurat
 }
 
 // process an array of tasks in parallel, using no more than vjobs worker threads
-fn (mut tasks []TaskDescription) run() {
+fn (mut tasks Tasks) run() {
-	vjobs := runtime.nr_jobs()
+	tasks.show_cmd = os.getenv('VTEST_SHOW_CMD') != ''
+	vjobs := if tasks.parallel_jobs > 0 { tasks.parallel_jobs } else { runtime.nr_jobs() }
 	mut bench := benchmark.new_benchmark()
-	bench.set_total_expected_steps(tasks.len)
+	bench.set_total_expected_steps(tasks.all.len)
-	mut work := sync.new_channel<TaskDescription>(tasks.len)
+	mut work := sync.new_channel<TaskDescription>(tasks.all.len)
-	mut results := sync.new_channel<TaskDescription>(tasks.len)
+	mut results := sync.new_channel<TaskDescription>(tasks.all.len)
 	mut m_skip_files := skip_files.clone()
 	if os.getenv('V_CI_UBUNTU_MUSL').len > 0 {
 		m_skip_files << skip_on_ubuntu_musl
@@ -109,18 +146,18 @@ fn (mut tasks []TaskDescription) run() {
 		m_skip_files << 'vlib/v/checker/tests/missing_c_lib_header_1.vv'
 		m_skip_files << 'vlib/v/checker/tests/missing_c_lib_header_with_explanation_2.vv'
 	}
-	for i in 0 .. tasks.len {
+	for i in 0 .. tasks.all.len {
-		if tasks[i].path in m_skip_files {
+		if tasks.all[i].path in m_skip_files {
-			tasks[i].is_skipped = true
+			tasks.all[i].is_skipped = true
 		}
-		unsafe { work.push(&tasks[i]) }
+		unsafe { work.push(&tasks.all[i]) }
 	}
 	work.close()
 	for _ in 0 .. vjobs {
 		go work_processor(mut work, mut results)
 	}
 	mut total_errors := 0
-	for _ in 0 .. tasks.len {
+	for _ in 0 .. tasks.all.len {
 		mut task := TaskDescription{}
 		results.pop(&task)
 		bench.step()
@@ -134,6 +171,9 @@ fn (mut tasks []TaskDescription) run() {
 			bench.fail()
 			eprintln(bstep_message(mut bench, benchmark.b_fail, task.path, task.took))
 			println('============')
+			println('failed cmd: $task.cli_cmd')
+			println('expected_out_path: $task.expected_out_path')
+			println('============')
 			println('expected:')
 			println(task.expected)
 			println('============')
@@ -143,12 +183,17 @@ fn (mut tasks []TaskDescription) run() {
 			diff_content(task.expected, task.found___)
 		} else {
 			bench.ok()
+			if tasks.show_cmd {
+				eprintln(bstep_message(mut bench, benchmark.b_ok, '$task.cli_cmd $task.path',
+					task.took))
+			} else {
 				eprintln(bstep_message(mut bench, benchmark.b_ok, task.path, task.took))
 			}
 		}
+	}
 	bench.stop()
 	eprintln(term.h_divider('-'))
-	eprintln(bench.total_message('all tests'))
+	eprintln(bench.total_message(tasks.label))
 	if total_errors != 0 {
 		exit(1)
 	}
@@ -178,6 +223,8 @@ fn (mut task TaskDescription) execute() {
 	cli_cmd := '$task.vexe $task.voptions $program'
 	res := os.exec(cli_cmd) or { panic(err) }
 	expected_out_path := program.replace('.vv', '') + task.result_extension
+	task.expected_out_path = expected_out_path
+	task.cli_cmd = cli_cmd
 	if should_autofix && !os.exists(expected_out_path) {
 		os.write_file(expected_out_path, '') or { panic(err) }
 	}

vlib/v/fmt/fmt.v

@@ -1147,6 +1147,8 @@ pub fn (mut f Fmt) comptime_call(node ast.ComptimeCall) {
 	} else {
 		if node.is_embed {
 			f.write("\$embed_file('$node.embed_file.rpath')")
+		} else if node.is_env {
+			f.write("\$env('$node.args_var')")
 		} else {
 			method_expr := if node.has_parens {
 				'(${node.method_name}($node.args_var))'

vlib/v/gen/comptime.v

@@ -3,6 +3,7 @@
 // that can be found in the LICENSE file.
 module gen
 
+import os
 import v.ast
 import v.table
 import v.util
@@ -29,9 +30,16 @@ fn (mut g Gen) comptime_selector(node ast.ComptimeSelector) {
 
 fn (mut g Gen) comptime_call(node ast.ComptimeCall) {
 	if node.is_embed {
+		// $embed_file('/path/to/file')
 		g.gen_embed_file_init(node)
 		return
 	}
+	if node.method_name == 'env' {
+		// $env('ENV_VAR_NAME')
+		val := util.cescaped_path(os.getenv(node.args_var))
+		g.write('_SLIT("$val")')
+		return
+	}
 	if node.is_vweb {
 		is_html := node.method_name == 'html'
 		for stmt in node.vweb_tmpl.stmts {

vlib/v/parser/comptime.v

@@ -11,7 +11,7 @@ import v.token
 import vweb.tmpl
 
 const (
-	supported_comptime_calls = ['html', 'tmpl', 'embed_file']
+	supported_comptime_calls = ['html', 'tmpl', 'env', 'embed_file']
 )
 
 // // #include, #flag, #v
@@ -47,7 +47,7 @@ fn (mut p Parser) comp_call() ast.ComptimeCall {
 		scope: 0
 	}
 	p.check(.dollar)
-	error_msg := 'only `\$tmpl()`, `\$embed_file()` and `\$vweb.html()` comptime functions are supported right now'
+	error_msg := 'only `\$tmpl()`, `\$env()`, `\$embed_file()` and `\$vweb.html()` comptime functions are supported right now'
 	if p.peek_tok.kind == .dot {
 		n := p.check_name() // skip `vweb.html()` TODO
 		if n != 'vweb' {
@@ -63,6 +63,21 @@ fn (mut p Parser) comp_call() ast.ComptimeCall {
 	}
 	is_embed_file := n == 'embed_file'
 	is_html := n == 'html'
+	// $env('ENV_VAR_NAME')
+	if n == 'env' {
+		p.check(.lpar)
+		spos := p.tok.position()
+		s := p.tok.lit
+		p.check(.string)
+		p.check(.rpar)
+		return ast.ComptimeCall{
+			scope: 0
+			method_name: n
+			args_var: s
+			is_env: true
+			env_pos: spos
+		}
+	}
 	p.check(.lpar)
 	spos := p.tok.position()
 	s := if is_html { '' } else { p.tok.lit }
@@ -70,12 +85,12 @@ fn (mut p Parser) comp_call() ast.ComptimeCall {
 		p.check(.string)
 	}
 	p.check(.rpar)
-	//
+	// $embed_file('/path/to/file')
 	if is_embed_file {
 		mut epath := s
 		// Validate that the epath exists, and that it is actually a file.
 		if epath == '' {
-			p.error_with_pos('please supply a valid relative or absolute file path to the file to embed',
+			p.error_with_pos('supply a valid relative or absolute file path to the file to embed',
 				spos)
 			return err_node
 		}

vlib/v/util/util.v

@@ -120,6 +120,49 @@ pub fn resolve_vroot(str string, dir string) ?string {
 	return str.replace('@VROOT', os.real_path(vmod_path))
 }
 
+// resolve_env_value replaces all occurrences of `$env('ENV_VAR_NAME')`
+// in `str` with the value of the env variable `$ENV_VAR_NAME`.
+pub fn resolve_env_value(str string, check_for_presence bool) ?string {
+	env_ident := "\$env('"
+	at := str.index(env_ident) or {
+		return error('no "$env_ident' + '...\')" could be found in "$str".')
+	}
+	mut ch := byte(`.`)
+	mut env_lit := ''
+	for i := at + env_ident.len; i < str.len && ch != `)`; i++ {
+		ch = byte(str[i])
+		if ch.is_letter() || ch.is_digit() || ch == `_` {
+			env_lit += ch.ascii_str()
+		} else {
+			if !(ch == `\'` || ch == `)`) {
+				if ch == `$` {
+					return error('cannot use string interpolation in compile time \$env() expression')
+				}
+				return error('invalid environment variable name in "$str", invalid character "$ch.ascii_str()"')
+			}
+		}
+	}
+	if env_lit == '' {
+		return error('supply an env variable name like HOME, PATH or USER')
+	}
+	mut env_value := ''
+	if check_for_presence {
+		env_value = os.environ()[env_lit] or {
+			return error('the environment variable "$env_lit" does not exist.')
+		}
+		if env_value == '' {
+			return error('the environment variable "$env_lit" is empty.')
+		}
+	} else {
+		env_value = os.getenv(env_lit)
+	}
+	rep := str.replace_once(env_ident + env_lit + "'" + ')', env_value)
+	if rep.contains(env_ident) {
+		return resolve_env_value(rep, check_for_presence)
+	}
+	return rep
+}
+
 // launch_tool - starts a V tool in a separate process, passing it the `args`.
 // All V tools are located in the cmd/tools folder, in files or folders prefixed by
 // the letter `v`, followed by the tool name, i.e. `cmd/tools/vdoc/` or `cmd/tools/vpm.v`.