diff --git a/CHANGELOG.md b/CHANGELOG.md index fcc1ae7eaf..4463e71614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ from local variables. - `__offsetof` for low level needs (works like `offsetof` in C). - vfmt now preserves empty lines, like gofmt. +- Support for compile time environment variables via `$env('ENV_VAR')`. ## V 0.2.1 *30 Dec 2020* diff --git a/cmd/tools/vtest.v b/cmd/tools/vtest.v index bd0a093662..727c6da2de 100644 --- a/cmd/tools/vtest.v +++ b/cmd/tools/vtest.v @@ -35,7 +35,7 @@ fn main() { } testing.header('Testing...') ts.test() - println(ts.benchmark.total_message('Ran all V _test.v files')) + println(ts.benchmark.total_message('all V _test.v files')) if ts.failed { exit(1) } diff --git a/doc/docs.md b/doc/docs.md index a292c29656..27d36d13a7 100644 --- a/doc/docs.md +++ b/doc/docs.md @@ -3346,6 +3346,21 @@ executable, increasing your binary size, but making it more self contained and thus easier to distribute. In this case, `f.data()` will cause *no IO*, and it will always return the same data. +#### $env + +```v +module main + +fn main() { + compile_time_env := $env('ENV_VAR') + println(compile_time_env) +} +``` + +V can bring in values at compile time from environment variables. +`$env('ENV_VAR')` can also be used in top-level `#flag` and `#include` statements: +`#flag linux -I $env('JAVA_HOME')/include`. + ### Environment specific files If a file has an environment-specific suffix, it will only be compiled for that environment. diff --git a/vlib/benchmark/benchmark.v b/vlib/benchmark/benchmark.v index dfb95ad350..ce8df9bd8b 100644 --- a/vlib/benchmark/benchmark.v +++ b/vlib/benchmark/benchmark.v @@ -189,7 +189,8 @@ pub fn (b &Benchmark) step_message_skip(msg string) string { // total_message returns a string with total summary of the benchmark run. pub fn (b &Benchmark) total_message(msg string) string { - mut tmsg := '${term.colorize(term.bold, 'Summary:')} ' + the_label := term.colorize(term.gray, msg) + mut tmsg := '${term.colorize(term.bold, 'Summary for $the_label:')} ' if b.nfail > 0 { tmsg += term.colorize(term.bold, term.colorize(term.red, '$b.nfail failed')) + ', ' } @@ -200,7 +201,6 @@ pub fn (b &Benchmark) total_message(msg string) string { tmsg += term.colorize(term.bold, term.colorize(term.yellow, '$b.nskip skipped')) + ', ' } tmsg += '$b.ntotal total. ${term.colorize(term.bold, 'Runtime:')} ${b.bench_timer.elapsed().microseconds() / 1000} ms.\n' - tmsg += term.colorize(term.gray, msg) return tmsg } diff --git a/vlib/v/ast/ast.v b/vlib/v/ast/ast.v index a86d9abfee..f8f787b62a 100644 --- a/vlib/v/ast/ast.v +++ b/vlib/v/ast/ast.v @@ -1133,14 +1133,20 @@ pub: method_pos token.Position scope &Scope left Expr - is_vweb bool - vweb_tmpl File args_var string - is_embed bool - embed_file EmbeddedFile + // + is_vweb bool + vweb_tmpl File + // + is_embed bool + embed_file EmbeddedFile + // + is_env bool + env_pos token.Position pub mut: sym table.TypeSymbol result_type table.Type + env_value string } pub struct None { diff --git a/vlib/v/checker/checker.v b/vlib/v/checker/checker.v index 4081d00c2a..ee41ee6144 100644 --- a/vlib/v/checker/checker.v +++ b/vlib/v/checker/checker.v @@ -3204,6 +3204,14 @@ fn (mut c Checker) hash_stmt(mut node ast.HashStmt) { } node.val = 'include $vroot' node.main = vroot + flag = vroot + } + if flag.contains('\$env(') { + env := util.resolve_env_value(flag, true) or { + c.error(err, node.pos) + return + } + node.main = env } flag_no_comment := flag.all_before('//').trim_space() if !((flag_no_comment.starts_with('"') && flag_no_comment.ends_with('"')) @@ -3239,6 +3247,12 @@ fn (mut c Checker) hash_stmt(mut node ast.HashStmt) { return } } + if flag.contains('\$env(') { + flag = util.resolve_env_value(flag, true) or { + c.error(err, node.pos) + return + } + } for deprecated in ['@VMOD', '@VMODULE', '@VPATH', '@VLIB_PATH'] { if flag.contains(deprecated) { c.error('$deprecated had been deprecated, use @VROOT instead.', node.pos) @@ -3663,6 +3677,14 @@ pub fn (mut c Checker) cast_expr(mut node ast.CastExpr) table.Type { fn (mut c Checker) comptime_call(mut node ast.ComptimeCall) table.Type { node.sym = c.table.get_type_symbol(c.unwrap_generic(c.expr(node.left))) + if node.is_env { + env_value := util.resolve_env_value("\$env('$node.args_var')", false) or { + c.error(err, node.env_pos) + return table.string_type + } + node.env_value = env_value + return table.string_type + } if node.is_embed { c.file.embedded_files << node.embed_file return c.table.find_type_idx('v.embed_file.EmbedFileData') diff --git a/vlib/v/checker/tests/comptime_env/env_parser_errors_1.run.out b/vlib/v/checker/tests/comptime_env/env_parser_errors_1.run.out new file mode 100644 index 0000000000..b66beac804 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/env_parser_errors_1.run.out @@ -0,0 +1,3 @@ +vlib/v/checker/tests/comptime_env/env_parser_errors_1.vv:1:3: error: supply an env variable name like HOME, PATH or USER + 1 | #flag -I $env('')/xyz + | ~~~~~~~~~~~~~~~~~~~ diff --git a/vlib/v/checker/tests/comptime_env/env_parser_errors_1.vv b/vlib/v/checker/tests/comptime_env/env_parser_errors_1.vv new file mode 100644 index 0000000000..83b2ff333f --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/env_parser_errors_1.vv @@ -0,0 +1 @@ +#flag -I $env('')/xyz diff --git a/vlib/v/checker/tests/comptime_env/env_parser_errors_2.run.out b/vlib/v/checker/tests/comptime_env/env_parser_errors_2.run.out new file mode 100644 index 0000000000..4c8cd7cac3 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/env_parser_errors_2.run.out @@ -0,0 +1,3 @@ +vlib/v/checker/tests/comptime_env/env_parser_errors_2.vv:1:3: error: cannot use string interpolation in compile time $env() expression + 1 | #flag -I $env('$ABC')/xyz + | ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/vlib/v/checker/tests/comptime_env/env_parser_errors_2.vv b/vlib/v/checker/tests/comptime_env/env_parser_errors_2.vv new file mode 100644 index 0000000000..abd0b76425 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/env_parser_errors_2.vv @@ -0,0 +1 @@ +#flag -I $env('$ABC')/xyz diff --git a/vlib/v/checker/tests/comptime_env/env_parser_errors_3.run.out b/vlib/v/checker/tests/comptime_env/env_parser_errors_3.run.out new file mode 100644 index 0000000000..df1f0cd5f3 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/env_parser_errors_3.run.out @@ -0,0 +1,3 @@ +vlib/v/checker/tests/comptime_env/env_parser_errors_3.vv:1:3: error: no "$env('...')" could be found in "-I $env()/xyz". + 1 | #flag -I $env()/xyz + | ~~~~~~~~~~~~~~~~~ diff --git a/vlib/v/checker/tests/comptime_env/env_parser_errors_3.vv b/vlib/v/checker/tests/comptime_env/env_parser_errors_3.vv new file mode 100644 index 0000000000..98fc6fe5be --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/env_parser_errors_3.vv @@ -0,0 +1 @@ +#flag -I $env()/xyz diff --git a/vlib/v/checker/tests/comptime_env/using_comptime_env.run.out b/vlib/v/checker/tests/comptime_env/using_comptime_env.run.out new file mode 100644 index 0000000000..a6e9193719 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/using_comptime_env.run.out @@ -0,0 +1,11 @@ +vlib/v/checker/tests/comptime_env/using_comptime_env.vv:1:3: error: the environment variable "VAR" does not exist. + 1 | #flag -I $env('VAR')/xyz + | ~~~~~~~~~~~~~~~~~~~~~~ + 2 | #include "$env('VAR')/stdio.h" + 3 | +vlib/v/checker/tests/comptime_env/using_comptime_env.vv:2:3: error: the environment variable "VAR" does not exist. + 1 | #flag -I $env('VAR')/xyz + 2 | #include "$env('VAR')/stdio.h" + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 3 | + 4 | fn main() { diff --git a/vlib/v/checker/tests/comptime_env/using_comptime_env.var.run.out b/vlib/v/checker/tests/comptime_env/using_comptime_env.var.run.out new file mode 100644 index 0000000000..012aa03cb3 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/using_comptime_env.var.run.out @@ -0,0 +1,2 @@ +/usr/include +done diff --git a/vlib/v/checker/tests/comptime_env/using_comptime_env.var_invalid.run.out b/vlib/v/checker/tests/comptime_env/using_comptime_env.var_invalid.run.out new file mode 100644 index 0000000000..0739bb2c43 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/using_comptime_env.var_invalid.run.out @@ -0,0 +1 @@ +builder error: '/opt/invalid/path/stdio.h' not found diff --git a/vlib/v/checker/tests/comptime_env/using_comptime_env.vv b/vlib/v/checker/tests/comptime_env/using_comptime_env.vv new file mode 100644 index 0000000000..f49508c061 --- /dev/null +++ b/vlib/v/checker/tests/comptime_env/using_comptime_env.vv @@ -0,0 +1,8 @@ +#flag -I $env('VAR')/xyz +#include "$env('VAR')/stdio.h" + +fn main() { + env := $env('VAR') + println(env) + println('done') +} diff --git a/vlib/v/compiler_errors_test.v b/vlib/v/compiler_errors_test.v index a0da40c625..fde5b85431 100644 --- a/vlib/v/compiler_errors_test.v +++ b/vlib/v/compiler_errors_test.v @@ -19,6 +19,8 @@ const turn_off_vcolors = os.setenv('VCOLORS', 'never', true) const should_autofix = os.getenv('VAUTOFIX') != '' +const github_job = os.getenv('GITHUB_JOB') + struct TaskDescription { vexe string dir string @@ -26,12 +28,23 @@ struct TaskDescription { result_extension string path string mut: - is_error bool - is_skipped bool - is_module bool - expected string - found___ string - took time.Duration + is_error bool + is_skipped bool + is_module bool + expected string + expected_out_path string + found___ string + took time.Duration + cli_cmd string +} + +struct Tasks { + vexe string + parallel_jobs int // 0 is using VJOBS, anything else is an override + label string +mut: + show_cmd bool + all []TaskDescription } fn test_all() { @@ -52,28 +65,51 @@ fn test_all() { module_tests := get_tests_in_dir(module_dir, true) run_tests := get_tests_in_dir(run_dir, false) // -prod is used for the parser and checker tests, so that warns are errors - mut tasks := []TaskDescription{} - tasks.add(vexe, parser_dir, '-prod', '.out', parser_tests, false) - tasks.add(vexe, checker_dir, '-prod', '.out', checker_tests, false) - tasks.add(vexe, scanner_dir, '-prod', '.out', scanner_tests, false) - tasks.add(vexe, checker_dir, '-d mysymbol run', '.mysymbol.run.out', ['custom_comptime_define_error.vv'], + mut tasks := Tasks{ + vexe: vexe + label: 'all tests' + } + tasks.add('', parser_dir, '-prod', '.out', parser_tests, false) + tasks.add('', checker_dir, '-prod', '.out', checker_tests, false) + tasks.add('', scanner_dir, '-prod', '.out', scanner_tests, false) + tasks.add('', checker_dir, '-d mysymbol run', '.mysymbol.run.out', ['custom_comptime_define_error.vv'], false) - tasks.add(vexe, checker_dir, '-d mydebug run', '.mydebug.run.out', ['custom_comptime_define_if_flag.vv'], + tasks.add('', checker_dir, '-d mydebug run', '.mydebug.run.out', ['custom_comptime_define_if_flag.vv'], false) - tasks.add(vexe, checker_dir, '-d nodebug run', '.nodebug.run.out', ['custom_comptime_define_if_flag.vv'], + tasks.add('', checker_dir, '-d nodebug run', '.nodebug.run.out', ['custom_comptime_define_if_flag.vv'], false) - tasks.add(vexe, checker_dir, '--enable-globals run', '.run.out', ['globals_error.vv'], + tasks.add('', checker_dir, '--enable-globals run', '.run.out', ['globals_error.vv'], false) - tasks.add(vexe, global_dir, '--enable-globals', '.out', global_tests, false) - tasks.add(vexe, module_dir, '-prod run', '.out', module_tests, true) - tasks.add(vexe, run_dir, 'run', '.run.out', run_tests, false) + tasks.add('', global_dir, '--enable-globals', '.out', global_tests, false) + tasks.add('', module_dir, '-prod run', '.out', module_tests, true) + tasks.add('', run_dir, 'run', '.run.out', run_tests, false) tasks.run() + if github_job == 'ubuntu-tcc' { + // these should be run serially, since they depend on setting and using environment variables + mut cte_tasks := Tasks{ + vexe: vexe + parallel_jobs: 1 + label: 'comptime env tests' + } + cte_dir := '$checker_dir/comptime_env' + files := get_tests_in_dir(cte_dir, false) + cte_tasks.add('', cte_dir, '-no-retry-compilation run', '.run.out', files, false) + cte_tasks.add('VAR=/usr/include $vexe', cte_dir, '-no-retry-compilation run', + '.var.run.out', ['using_comptime_env.vv'], false) + cte_tasks.add('VAR=/opt/invalid/path $vexe', cte_dir, '-no-retry-compilation run', + '.var_invalid.run.out', ['using_comptime_env.vv'], false) + cte_tasks.run() + } } -fn (mut tasks []TaskDescription) add(vexe string, dir string, voptions string, result_extension string, tests []string, is_module bool) { +fn (mut tasks Tasks) add(custom_vexe string, dir string, voptions string, result_extension string, tests []string, is_module bool) { + mut vexe := tasks.vexe + if custom_vexe != '' { + vexe = custom_vexe + } paths := vtest.filter_vtest_only(tests, basepath: dir) for path in paths { - tasks << TaskDescription{ + tasks.all << TaskDescription{ vexe: vexe dir: dir voptions: voptions @@ -89,12 +125,13 @@ fn bstep_message(mut bench benchmark.Benchmark, label string, msg string, sdurat } // process an array of tasks in parallel, using no more than vjobs worker threads -fn (mut tasks []TaskDescription) run() { - vjobs := runtime.nr_jobs() +fn (mut tasks Tasks) run() { + tasks.show_cmd = os.getenv('VTEST_SHOW_CMD') != '' + vjobs := if tasks.parallel_jobs > 0 { tasks.parallel_jobs } else { runtime.nr_jobs() } mut bench := benchmark.new_benchmark() - bench.set_total_expected_steps(tasks.len) - mut work := sync.new_channel(tasks.len) - mut results := sync.new_channel(tasks.len) + bench.set_total_expected_steps(tasks.all.len) + mut work := sync.new_channel(tasks.all.len) + mut results := sync.new_channel(tasks.all.len) mut m_skip_files := skip_files.clone() if os.getenv('V_CI_UBUNTU_MUSL').len > 0 { m_skip_files << skip_on_ubuntu_musl @@ -109,18 +146,18 @@ fn (mut tasks []TaskDescription) run() { m_skip_files << 'vlib/v/checker/tests/missing_c_lib_header_1.vv' m_skip_files << 'vlib/v/checker/tests/missing_c_lib_header_with_explanation_2.vv' } - for i in 0 .. tasks.len { - if tasks[i].path in m_skip_files { - tasks[i].is_skipped = true + for i in 0 .. tasks.all.len { + if tasks.all[i].path in m_skip_files { + tasks.all[i].is_skipped = true } - unsafe { work.push(&tasks[i]) } + unsafe { work.push(&tasks.all[i]) } } work.close() for _ in 0 .. vjobs { go work_processor(mut work, mut results) } mut total_errors := 0 - for _ in 0 .. tasks.len { + for _ in 0 .. tasks.all.len { mut task := TaskDescription{} results.pop(&task) bench.step() @@ -134,6 +171,9 @@ fn (mut tasks []TaskDescription) run() { bench.fail() eprintln(bstep_message(mut bench, benchmark.b_fail, task.path, task.took)) println('============') + println('failed cmd: $task.cli_cmd') + println('expected_out_path: $task.expected_out_path') + println('============') println('expected:') println(task.expected) println('============') @@ -143,12 +183,17 @@ fn (mut tasks []TaskDescription) run() { diff_content(task.expected, task.found___) } else { bench.ok() - eprintln(bstep_message(mut bench, benchmark.b_ok, task.path, task.took)) + if tasks.show_cmd { + eprintln(bstep_message(mut bench, benchmark.b_ok, '$task.cli_cmd $task.path', + task.took)) + } else { + eprintln(bstep_message(mut bench, benchmark.b_ok, task.path, task.took)) + } } } bench.stop() eprintln(term.h_divider('-')) - eprintln(bench.total_message('all tests')) + eprintln(bench.total_message(tasks.label)) if total_errors != 0 { exit(1) } @@ -178,6 +223,8 @@ fn (mut task TaskDescription) execute() { cli_cmd := '$task.vexe $task.voptions $program' res := os.exec(cli_cmd) or { panic(err) } expected_out_path := program.replace('.vv', '') + task.result_extension + task.expected_out_path = expected_out_path + task.cli_cmd = cli_cmd if should_autofix && !os.exists(expected_out_path) { os.write_file(expected_out_path, '') or { panic(err) } } diff --git a/vlib/v/fmt/fmt.v b/vlib/v/fmt/fmt.v index 8ec02ed997..70922ae229 100644 --- a/vlib/v/fmt/fmt.v +++ b/vlib/v/fmt/fmt.v @@ -1147,6 +1147,8 @@ pub fn (mut f Fmt) comptime_call(node ast.ComptimeCall) { } else { if node.is_embed { f.write("\$embed_file('$node.embed_file.rpath')") + } else if node.is_env { + f.write("\$env('$node.args_var')") } else { method_expr := if node.has_parens { '(${node.method_name}($node.args_var))' diff --git a/vlib/v/gen/comptime.v b/vlib/v/gen/comptime.v index 0e3870c0ba..c94183122d 100644 --- a/vlib/v/gen/comptime.v +++ b/vlib/v/gen/comptime.v @@ -3,6 +3,7 @@ // that can be found in the LICENSE file. module gen +import os import v.ast import v.table import v.util @@ -29,9 +30,16 @@ fn (mut g Gen) comptime_selector(node ast.ComptimeSelector) { fn (mut g Gen) comptime_call(node ast.ComptimeCall) { if node.is_embed { + // $embed_file('/path/to/file') g.gen_embed_file_init(node) return } + if node.method_name == 'env' { + // $env('ENV_VAR_NAME') + val := util.cescaped_path(os.getenv(node.args_var)) + g.write('_SLIT("$val")') + return + } if node.is_vweb { is_html := node.method_name == 'html' for stmt in node.vweb_tmpl.stmts { diff --git a/vlib/v/parser/comptime.v b/vlib/v/parser/comptime.v index b465c335a7..ed0aa5e8e3 100644 --- a/vlib/v/parser/comptime.v +++ b/vlib/v/parser/comptime.v @@ -11,7 +11,7 @@ import v.token import vweb.tmpl const ( - supported_comptime_calls = ['html', 'tmpl', 'embed_file'] + supported_comptime_calls = ['html', 'tmpl', 'env', 'embed_file'] ) // // #include, #flag, #v @@ -47,7 +47,7 @@ fn (mut p Parser) comp_call() ast.ComptimeCall { scope: 0 } p.check(.dollar) - error_msg := 'only `\$tmpl()`, `\$embed_file()` and `\$vweb.html()` comptime functions are supported right now' + error_msg := 'only `\$tmpl()`, `\$env()`, `\$embed_file()` and `\$vweb.html()` comptime functions are supported right now' if p.peek_tok.kind == .dot { n := p.check_name() // skip `vweb.html()` TODO if n != 'vweb' { @@ -63,6 +63,21 @@ fn (mut p Parser) comp_call() ast.ComptimeCall { } is_embed_file := n == 'embed_file' is_html := n == 'html' + // $env('ENV_VAR_NAME') + if n == 'env' { + p.check(.lpar) + spos := p.tok.position() + s := p.tok.lit + p.check(.string) + p.check(.rpar) + return ast.ComptimeCall{ + scope: 0 + method_name: n + args_var: s + is_env: true + env_pos: spos + } + } p.check(.lpar) spos := p.tok.position() s := if is_html { '' } else { p.tok.lit } @@ -70,12 +85,12 @@ fn (mut p Parser) comp_call() ast.ComptimeCall { p.check(.string) } p.check(.rpar) - // + // $embed_file('/path/to/file') if is_embed_file { mut epath := s // Validate that the epath exists, and that it is actually a file. if epath == '' { - p.error_with_pos('please supply a valid relative or absolute file path to the file to embed', + p.error_with_pos('supply a valid relative or absolute file path to the file to embed', spos) return err_node } diff --git a/vlib/v/util/util.v b/vlib/v/util/util.v index 12003a2992..cb99d18862 100644 --- a/vlib/v/util/util.v +++ b/vlib/v/util/util.v @@ -120,6 +120,49 @@ pub fn resolve_vroot(str string, dir string) ?string { return str.replace('@VROOT', os.real_path(vmod_path)) } +// resolve_env_value replaces all occurrences of `$env('ENV_VAR_NAME')` +// in `str` with the value of the env variable `$ENV_VAR_NAME`. +pub fn resolve_env_value(str string, check_for_presence bool) ?string { + env_ident := "\$env('" + at := str.index(env_ident) or { + return error('no "$env_ident' + '...\')" could be found in "$str".') + } + mut ch := byte(`.`) + mut env_lit := '' + for i := at + env_ident.len; i < str.len && ch != `)`; i++ { + ch = byte(str[i]) + if ch.is_letter() || ch.is_digit() || ch == `_` { + env_lit += ch.ascii_str() + } else { + if !(ch == `\'` || ch == `)`) { + if ch == `$` { + return error('cannot use string interpolation in compile time \$env() expression') + } + return error('invalid environment variable name in "$str", invalid character "$ch.ascii_str()"') + } + } + } + if env_lit == '' { + return error('supply an env variable name like HOME, PATH or USER') + } + mut env_value := '' + if check_for_presence { + env_value = os.environ()[env_lit] or { + return error('the environment variable "$env_lit" does not exist.') + } + if env_value == '' { + return error('the environment variable "$env_lit" is empty.') + } + } else { + env_value = os.getenv(env_lit) + } + rep := str.replace_once(env_ident + env_lit + "'" + ')', env_value) + if rep.contains(env_ident) { + return resolve_env_value(rep, check_for_presence) + } + return rep +} + // launch_tool - starts a V tool in a separate process, passing it the `args`. // All V tools are located in the cmd/tools folder, in files or folders prefixed by // the letter `v`, followed by the tool name, i.e. `cmd/tools/vdoc/` or `cmd/tools/vpm.v`.