From b1a9bf29db157274f4475dfb2353cde8c40be4a2 Mon Sep 17 00:00:00 2001 From: Larpon Date: Mon, 13 Dec 2021 19:58:31 +0100 Subject: [PATCH] vgret: add support for config via `toml` and root path (#12821) --- .github/workflows/gfx_ci.yml | 2 +- cmd/tools/vgret.defaults.toml | 74 ++++++++ cmd/tools/vgret.v | 326 ++++++++++++++++++++++------------ 3 files changed, 291 insertions(+), 111 deletions(-) create mode 100644 cmd/tools/vgret.defaults.toml diff --git a/.github/workflows/gfx_ci.yml b/.github/workflows/gfx_ci.yml index 8af83c641e..a7a31b3f3c 100644 --- a/.github/workflows/gfx_ci.yml +++ b/.github/workflows/gfx_ci.yml @@ -44,7 +44,7 @@ jobs: continue-on-error: true run: | Xvfb $DISPLAY -screen 0 1280x1024x24 & - ./v gret -v ./gg-sample_images ./gg-regression-images + ./v gret -t ./gg-regression-images/vgret.v_examples.toml -v ./gg-sample_images ./gg-regression-images - name: Upload regression to imgur if: steps.compare.outcome != 'success' diff --git a/cmd/tools/vgret.defaults.toml b/cmd/tools/vgret.defaults.toml new file mode 100644 index 0000000000..43c80c0ef5 --- /dev/null +++ b/cmd/tools/vgret.defaults.toml @@ -0,0 +1,74 @@ +# Defaults +[compare] + method = 'idiff' + flags = ['-p','-fail 0.001','-failpercent 0.2'] + +[capture] + method = 'gg_record' + flags = [] # ['-prod','-d ...'] etc. + +[capture.env] + VGG_STOP_AT_FRAME = '8' + VGG_SCREENSHOT_FOLDER = '$OUT_PATH' + VGG_SCREENSHOT_FRAMES = '5' + +# List of apps to run and capture +[[apps]] + path = 'examples/game_of_life/life_gg.v' + +[[apps]] + path = 'examples/gg/bezier.v' + +[[apps]] + path = 'examples/gg/mandelbrot.v' + +[[apps]] + path = 'examples/gg/rectangles.v' + +[[apps]] + path = 'examples/gg/raven_text_rendering.v' + +[[apps]] + path = 'examples/gg/worker_thread.v' + +[[apps]] + path = 'examples/gg/polygons.v' + +[[apps]] + path = 'examples/gg/bezier_anim.v' + +[[apps]] + path = 'examples/gg/drag_n_drop.v' + +[[apps]] + path = 'examples/ttf_font/example_ttf.v' + +# Reasons for ex- or inclusion: +# +# 'examples/snek/snek.v' // Inacurrate captures +# 'examples/game_of_life/life_gg.v' // OK +# 'examples/tetris/tetris.v' // Uses random start block +# 'examples/fireworks/fireworks.v' // Uses rand for placement +# 'examples/gg/bezier.v', // OK +# 'examples/gg/mandelbrot.v', // OK +# 'examples/gg/rectangles.v', // OK +# 'examples/gg/set_pixels.v' // Has problem in CI software render (blank, no pixels set) +# 'examples/gg/random.v' // Always random +# 'examples/gg/stars.v' // Uses rand for placement +# 'examples/gg/raven_text_rendering.v', // OK +# 'examples/gg/worker_thread.v', // OK +# 'examples/gg/polygons.v', // OK +# 'examples/gg/bezier_anim.v', // OK +# 'examples/gg/drag_n_drop.v' // OK +# 'examples/2048/2048.v' // Random start tiles +# 'examples/clock/clock.v' // Can only be tested on exact points in time :) +# 'examples/flappylearning/game.v' // Random movement +# 'examples/hot_reload/bounce.v' // Inacurrate captures +# 'examples/hot_reload/graph.v' // Inacurrate captures +# 'examples/ttf_font/example_ttf.v', // OK +# 'examples/sokol/01_cubes/cube.v', // Can pass with a warning and diff at around 1.2% +# 'examples/sokol/02_cubes_glsl/cube_glsl.v', // Inacurrate captures +# 'examples/sokol/03_march_tracing_glsl/rt_glsl.v', // Inacurrate captures +# 'examples/sokol/04_multi_shader_glsl/rt_glsl.v', // Inacurrate captures +# 'examples/sokol/05_instancing_glsl/rt_glsl.v', // Inacurrate captures +# 'examples/sokol/06_obj_viewer/show_obj.v', // Inacurrate captures diff --git a/cmd/tools/vgret.v b/cmd/tools/vgret.v index a77bfe11bf..50c254f397 100644 --- a/cmd/tools/vgret.v +++ b/cmd/tools/vgret.v @@ -34,6 +34,7 @@ // import os import flag +import toml const ( tool_name = os.file_name(os.executable()) @@ -51,35 +52,6 @@ Examples: tmp_dir = os.join_path(os.temp_dir(), 'v', tool_name) runtime_os = os.user_os() v_root = os.real_path(@VMODROOT) - build_list = [ - //'examples/snek/snek.v' // Inacurrate captures - 'examples/game_of_life/life_gg.v' - //'examples/tetris/tetris.v' // Uses random start tile - //'examples/fireworks/fireworks.v' // Uses rand for placement - 'examples/gg/bezier.v', - 'examples/gg/mandelbrot.v', - 'examples/gg/rectangles.v', - //'examples/gg/set_pixels.v' // Has problem in CI software render (blank, no pixels set) - //'examples/gg/random.v' // Always random - //'examples/gg/stars.v' // Uses rand for placement - 'examples/gg/raven_text_rendering.v', - 'examples/gg/worker_thread.v', - 'examples/gg/polygons.v', - 'examples/gg/bezier_anim.v', - 'examples/gg/drag_n_drop.v' - //'examples/clock/clock.v' // Can only be tested on exact points in time :) - //'examples/hot_reload/bounce.v' // Inacurrate captures - //'examples/hot_reload/graph.v' // Inacurrate captures - //'examples/flappylearning/game.v' // Random movement - //'examples/2048/2048.v' // Random start tiles - 'examples/ttf_font/example_ttf.v', - //'examples/sokol/01_cubes/cube.v', // Can pass with a warning and diff at around 1.2% - //'examples/sokol/02_cubes_glsl/cube_glsl.v', // Inacurrate captures - //'examples/sokol/03_march_tracing_glsl/rt_glsl.v', // Inacurrate captures - //'examples/sokol/04_multi_shader_glsl/rt_glsl.v', // Inacurrate captures - //'examples/sokol/05_instancing_glsl/rt_glsl.v', // Inacurrate captures - //'examples/sokol/06_obj_viewer/show_obj.v', // Inacurrate captures - ] ) const ( @@ -89,10 +61,48 @@ const ( idiff_exe = os.find_abs_path_of_executable('idiff') or { '' } ) +const ( + embedded_toml = $embed_file('vgret.defaults.toml', .zlib) + default_toml = embedded_toml.to_string() + empty_toml_array = []toml.Any{} + empty_toml_map = map[string]toml.Any{} +) + +struct Config { + path string +mut: + apps []AppConfig +} + +struct CompareOptions { +mut: + method string = 'idiff' + flags []string +} + +struct CaptureOptions { +mut: + method string = 'gg_record' + flags []string + env map[string]string +} + +struct AppConfig { + compare CompareOptions + capture CaptureOptions + path string + abs_path string +mut: + screenshots_path string + screenshots []string +} + struct Options { - show_help bool verbose bool compare_only bool + root_path string +mut: + config Config } fn main() { @@ -106,36 +116,56 @@ fn main() { fp.description(tool_description) fp.arguments_description('PATH [PATH]') fp.skip_executable() - // Collect tool options - opt := Options{ - show_help: fp.bool('help', `h`, false, 'Show this help text.') - verbose: fp.bool('verbose', `v`, false, "Be verbose about the tool's progress.") - compare_only: fp.bool('compare-only', `c`, false, "Don't generate screenshots - only compare input directories") - } - if opt.show_help { + + show_help := fp.bool('help', `h`, false, 'Show this help text.') + if show_help { println(fp.usage()) exit(0) } + // Collect tool options + mut opt := Options{ + verbose: fp.bool('verbose', `v`, false, "Be verbose about the tool's progress.") + compare_only: fp.bool('compare-only', `c`, false, "Don't generate screenshots - only compare input directories") + root_path: fp.string('root-path', `r`, v_root, 'Root path of the comparison') + } + + toml_conf := fp.string('toml-config', `t`, default_toml, 'Path or string with TOML configuration') + ensure_env(opt) or { panic(err) } arg_paths := fp.finalize() or { panic(err) } if arg_paths.len == 0 { println(fp.usage()) - println('Error missing arguments') + println('\nError missing arguments') exit(1) } - if arg_paths.len == 1 { - generate_screenshots(opt, v_root, arg_paths[0]) ? - } else if arg_paths.len > 1 { - compare_screenshots(opt, v_root, arg_paths[0], arg_paths[1]) or { panic(err) } + + opt.config = new_config(opt.root_path, toml_conf) ? + + gen_in_path := arg_paths[0] + if arg_paths.len >= 1 { + generate_screenshots(mut opt, gen_in_path) ? + } + if arg_paths.len > 1 { + target_path := arg_paths[1] + path := opt.config.path + all_paths_in_use := [path, gen_in_path, target_path] + for path_in_use in all_paths_in_use { + if !os.is_dir(path_in_use) { + panic('`$path_in_use` is not a directory') + } + } + if path == target_path || gen_in_path == target_path || gen_in_path == path { + panic('Compare paths can not be the same directory `$path`/`$target_path`/`$gen_in_path`') + } + compare_screenshots(opt, gen_in_path, target_path) or { panic(err) } } } -fn generate_screenshots(opt Options, base_path string, output_path string) ?[]string { - mut path := os.real_path(base_path) - path = path.trim_right('/') +fn generate_screenshots(mut opt Options, output_path string) ? { + path := opt.config.path dst_path := output_path.trim_right('/') @@ -143,16 +173,15 @@ fn generate_screenshots(opt Options, base_path string, output_path string) ?[]st return error('`$path` is not a directory') } - mut screenshots := []string{} - - for file in build_list { - app_path := os.join_path(path, file).trim_right('/') + for mut app_config in opt.config.apps { + file := app_config.path + app_path := app_config.abs_path mut rel_out_path := '' if os.is_file(app_path) { - rel_out_path = os.dir(file.trim_right('/')) + rel_out_path = os.dir(file) } else { - rel_out_path = file.trim_right('/') + rel_out_path = file } if opt.verbose { @@ -180,63 +209,52 @@ fn generate_screenshots(opt Options, base_path string, output_path string) ?[]st } } - screenshots << take_screenshots(opt, app_path, screenshot_path) or { - return error('Failed taking screenshot of `$app_path`:\n$err.msg') + app_config.screenshots_path = screenshot_path + app_config.screenshots = take_screenshots(opt, app_config) or { + return error('Failed taking screenshots of `$app_path`:\n$err.msg') } } - return screenshots } -fn compare_screenshots(opt Options, base_path string, output_path string, target_path string) ? { - if idiff_exe == '' { - return error('$tool_name need the `idiff` tool installed. It can be installed on Ubuntu with `sudo apt install openimageio-tools`') - } - - mut path := os.real_path(base_path) - path = path.trim_right('/') - - if !os.is_dir(path) { - return error('`$path` is not a directory') - } - if !os.is_dir(target_path) { - return error('`$target_path` is not a directory') - } - if path == target_path { - return error('Compare paths can not be the same directory `$path`') - } - - screenshots := generate_screenshots(opt, path, output_path) ? - - if opt.verbose { - eprintln('Comparing $screenshots.len screenshots in `$output_path` with `$target_path`') - } - +fn compare_screenshots(opt Options, output_path string, target_path string) ? { mut fails := map[string]string{} mut warns := map[string]string{} - for screenshot in screenshots { - relative_screenshot := screenshot.all_after(output_path + os.path_separator) - - src := screenshot - target := os.join_path(target_path, relative_screenshot) - + for app_config in opt.config.apps { + screenshots := app_config.screenshots if opt.verbose { - eprintln('Comparing `$src` with `$target`') + eprintln('Comparing $screenshots.len screenshots in `$output_path` with `$target_path`') } + for screenshot in screenshots { + relative_screenshot := screenshot.all_after(output_path + os.path_separator) - diff_file := os.join_path(os.temp_dir(), os.file_name(src).all_before_last('.') + - '.diff.tif') - diff_cmd := '$idiff_exe -p -fail 0.001 -failpercent 0.2 -od -o "$diff_file" -abs "$src" "$target"' - result := os.execute(diff_cmd) - if opt.verbose && result.exit_code == 0 { - eprintln('Running: $diff_cmd') - eprintln('$result.output') - } - if result.exit_code != 0 { - eprintln('$result.output') - if result.exit_code == 1 { - warns[src] = target - } else { - fails[src] = target + src := screenshot + target := os.join_path(target_path, relative_screenshot) + + if opt.verbose { + eprintln('Comparing `$src` with `$target` with $app_config.compare.method') + } + + if app_config.compare.method == 'idiff' { + if idiff_exe == '' { + return error('$tool_name need the `idiff` tool installed. It can be installed on Ubuntu with `sudo apt install openimageio-tools`') + } + diff_file := os.join_path(os.temp_dir(), os.file_name(src).all_before_last('.') + + '.diff.tif') + flags := app_config.compare.flags.join(' ') + diff_cmd := '$idiff_exe $flags -od -o "$diff_file" -abs "$src" "$target"' + result := os.execute(diff_cmd) + if opt.verbose && result.exit_code == 0 { + eprintln('Running: $diff_cmd') + eprintln('$result.output') + } + if result.exit_code != 0 { + eprintln('$result.output') + if result.exit_code == 1 { + warns[src] = target + } else { + fails[src] = target + } + } } } } @@ -262,29 +280,45 @@ fn compare_screenshots(opt Options, base_path string, output_path string, target diff_file := os.join_path(os.temp_dir(), os.file_name(first).all_before_last('.') + '.diff.tif') diff_copy := os.join_path(os.temp_dir(), 'diff.tif') - os.cp(diff_file, diff_copy) or { panic(err) } - eprintln('First failed diff file `$diff_file` is copied to `$diff_copy`') + if os.is_file(diff_file) { + os.cp(diff_file, diff_copy) or { panic(err) } + eprintln('First failed diff file `$diff_file` is copied to `$diff_copy`') + } exit(1) } } -fn take_screenshots(opt Options, app string, out_path string) ?[]string { +fn take_screenshots(opt Options, app AppConfig) ?[]string { + out_path := app.screenshots_path if !opt.compare_only { if opt.verbose { - eprintln('Taking screenshot(s) of `$app` to `$out_path`') + eprintln('Taking screenshot(s) of `$app.path` to `$out_path`') } - os.setenv('VGG_STOP_AT_FRAME', '8', true) - os.setenv('VGG_SCREENSHOT_FOLDER', out_path, true) - os.setenv('VGG_SCREENSHOT_FRAMES', '5', true) - result := os.execute('$v_exe -d gg_record run "$app"') - if result.exit_code != 0 { - return error('Failed taking screenshot of `$app`:\n$result.output') + + if app.capture.method == 'gg_record' { + for k, v in app.capture.env { + rv := v.replace('\$OUT_PATH', out_path) + if opt.verbose { + eprintln('Setting ENV `$k` = $rv ...') + } + os.setenv('$k', rv, true) + } + + mut flags := app.capture.flags.join(' ') + v_cmd := '$v_exe $flags -d gg_record run "$app.abs_path"' + if opt.verbose { + eprintln('Running `$v_cmd`') + } + result := os.execute('$v_cmd') + if result.exit_code != 0 { + return error('Failed taking screenshot of `$app.abs_path`:\n$result.output') + } } } mut screenshots := []string{} shots := os.ls(out_path) or { return error('Failed listing dir `$out_path`') } for shot in shots { - if shot.starts_with(os.file_name(app).all_before_last('.')) { + if shot.starts_with(os.file_name(app.path).all_before_last('.')) { screenshots << os.join_path(out_path, shot) } } @@ -314,3 +348,75 @@ fn vexe() string { } return exe } + +fn new_config(root_path string, toml_config string) ?Config { + doc := toml.parse(toml_config) ? + + path := os.real_path(root_path).trim_right('/') + + compare_method := doc.value('compare.method').default_to('idiff').string() + compare_flags := doc.value('compare.flags').default_to(empty_toml_array).array().as_strings() + default_compare := CompareOptions{ + method: compare_method + flags: compare_flags + } + capture_method := doc.value('capture.method').default_to('gg_record').string() + capture_flags := doc.value('capture.flags').default_to(empty_toml_array).array().as_strings() + capture_env := doc.value('capture.env').default_to(empty_toml_map).as_map() + mut env_map := map[string]string{} + for k, v in capture_env { + env_map[k] = v.string() + } + default_capture := CaptureOptions{ + method: capture_method + flags: capture_flags + env: env_map + } + + apps_any := doc.value('apps').default_to(empty_toml_array).array() + mut apps := []AppConfig{cap: apps_any.len} + for app_any in apps_any { + rel_path := app_any.value('path').string().trim_right('/') + + // Merge, per app, overwrites + mut merged_compare := CompareOptions{} + merged_compare.method = app_any.value('compare.method').default_to(default_compare.method).string() + merged_compare_flags := app_any.value('compare.flags').default_to(empty_toml_array).array().as_strings() + if merged_compare_flags.len > 0 { + merged_compare.flags = merged_compare_flags + } else { + merged_compare.flags = default_compare.flags + } + + mut merged_capture := CaptureOptions{} + merged_capture.method = app_any.value('capture.method').default_to(default_capture.method).string() + merged_capture_flags := app_any.value('capture.flags').default_to(empty_toml_array).array().as_strings() + if merged_capture_flags.len > 0 { + merged_capture.flags = merged_capture_flags + } else { + merged_capture.flags = default_capture.flags + } + + merge_capture_env := app_any.value('capture.env').default_to(empty_toml_map).as_map() + mut merge_env_map := default_capture.env.clone() + for k, v in merge_capture_env { + merge_env_map[k] = v.string() + } + for k, v in merge_env_map { + merged_capture.env[k] = v + } + + app_config := AppConfig{ + compare: merged_compare + capture: merged_capture + path: rel_path + abs_path: os.join_path(path, rel_path).trim_right('/') + } + apps << app_config + } + + return Config{ + apps: apps + path: path + } +}