From ea1f398f903c66353bb036025edbf4fc01d5ba6b Mon Sep 17 00:00:00 2001 From: Larpon Date: Sat, 11 Dec 2021 08:30:06 +0100 Subject: [PATCH] ci: add `gfx_ci` utilizing `vgret` (#12785) --- .github/workflows/gfx_ci.yml | 53 ++++++++++++++++++++++++++++ cmd/tools/vgret.v | 68 +++++++++++++++++++++++++++++------- 2 files changed, 109 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/gfx_ci.yml diff --git a/.github/workflows/gfx_ci.yml b/.github/workflows/gfx_ci.yml new file mode 100644 index 0000000000..8af83c641e --- /dev/null +++ b/.github/workflows/gfx_ci.yml @@ -0,0 +1,53 @@ +name: Graphics CI + +on: + push: + paths-ignore: + - "**.md" + pull_request: + paths-ignore: + - "**.md" + +jobs: + gg-regressions: + runs-on: ubuntu-18.04 + timeout-minutes: 10 + env: + VFLAGS: -cc tcc + DISPLAY: :99 + steps: + - name: Checkout V + uses: actions/checkout@v2 + + - name: Build local v + run: make -j4 + + - name: Setup dependencies + run: | + # xvfb : xvfb (installed by openrndr/setup-opengl@v1.1) + # openimageio-tools : idiff + # libxcursor-dev libxi-dev : V gfx deps + # mesa-common-dev : For headless rendering + # freeglut3-dev : Fixes graphic apps compilation with tcc + sudo apt-get install imagemagick openimageio-tools mesa-common-dev libxcursor-dev libxi-dev freeglut3-dev + wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh + chmod +x ./imgur.sh + + - uses: openrndr/setup-opengl@v1.1 + - uses: actions/checkout@v2 + with: + repository: Larpon/gg-regression-images + path: gg-regression-images + + - name: Sample and compare + id: compare + continue-on-error: true + run: | + Xvfb $DISPLAY -screen 0 1280x1024x24 & + ./v gret -v ./gg-sample_images ./gg-regression-images + + - name: Upload regression to imgur + if: steps.compare.outcome != 'success' + run: | + ./imgur.sh /tmp/fail.png + exit 1 diff --git a/cmd/tools/vgret.v b/cmd/tools/vgret.v index 113a7074b7..a77bfe11bf 100644 --- a/cmd/tools/vgret.v +++ b/cmd/tools/vgret.v @@ -14,6 +14,24 @@ // For developers: // For a quick overview of the generated images you can use `montage` from imagemagick to generate a "Contact Sheet": // montage -verbose -label '%f' -font Helvetica -pointsize 10 -background '#000000' -fill 'gray' -define jpeg:size=200x200 -geometry 200x200+2+2 -auto-orient $(fd -t f . /path/to/vgret/out/dir) /tmp/montage.jpg +// +// To generate the reference images locally - or for uploading to a remote repo like `gg-regression-images` +// You can do the following: +// 1. `export DISPLAY=:99` # Start all graphical apps on DISPLAY 99 +// 2. `Xvfb $DISPLAY -screen 0 1280x1024x24 &` # Starts a virtual X11 screen buffer +// 3. `v gret -v /tmp/gg-regression-images` # Generate reference images to /tmp/gg-regression-images +// 4. `v gret -v /tmp/test /tmp/gg-regression-images` # Test if the tests can pass locally by comparing to a fresh imageset +// 5. Visually check the images (you can get an overview by running the `montage` command above) +// 6. Upload to GitHub or keep locally for more testing/tweaking +// +// It's a known factor that the images generated on a local machine won't match the images generated on a remote machine by 100%. +// They will most likely differ by a small percentage - the comparison tool can be tweaked to accept these subtle changes, +// at the expense of slightly more inaccurate test results. For non-animated apps the percentage should be > 0.01. +// You can emulate or test these inaccuracies to some extend locally by simply running the test from a terminal using +// your physical X11 session display (Usually DISPLAY=:0). +// +// Read more about the options of `idiff` here: https://openimageio.readthedocs.io/en/latest/idiff.html +// import os import flag @@ -41,7 +59,7 @@ Examples: 'examples/gg/bezier.v', 'examples/gg/mandelbrot.v', 'examples/gg/rectangles.v', - 'examples/gg/set_pixels.v' + //'examples/gg/set_pixels.v' // Has problem in CI software render (blank, no pixels set) //'examples/gg/random.v' // Always random //'examples/gg/stars.v' // Uses rand for placement 'examples/gg/raven_text_rendering.v', @@ -54,13 +72,13 @@ Examples: //'examples/hot_reload/graph.v' // Inacurrate captures //'examples/flappylearning/game.v' // Random movement //'examples/2048/2048.v' // Random start tiles - 'examples/ttf_font/example_ttf.v' - //'examples/sokol/06_obj_viewer/show_obj.v' // Inacurrate captures - //'examples/sokol/04_multi_shader_glsl/rt_glsl.v' // Inacurrate captures - //'examples/sokol/03_march_tracing_glsl/rt_glsl.v' // Inacurrate captures - //'examples/sokol/02_cubes_glsl/cube_glsl.v' // Inacurrate captures - //'examples/sokol/05_instancing_glsl/rt_glsl.v' // Inacurrate captures - 'examples/sokol/01_cubes/cube.v', + 'examples/ttf_font/example_ttf.v', + //'examples/sokol/01_cubes/cube.v', // Can pass with a warning and diff at around 1.2% + //'examples/sokol/02_cubes_glsl/cube_glsl.v', // Inacurrate captures + //'examples/sokol/03_march_tracing_glsl/rt_glsl.v', // Inacurrate captures + //'examples/sokol/04_multi_shader_glsl/rt_glsl.v', // Inacurrate captures + //'examples/sokol/05_instancing_glsl/rt_glsl.v', // Inacurrate captures + //'examples/sokol/06_obj_viewer/show_obj.v', // Inacurrate captures ] ) @@ -194,6 +212,7 @@ fn compare_screenshots(opt Options, base_path string, output_path string, target } mut fails := map[string]string{} + mut warns := map[string]string{} for screenshot in screenshots { relative_screenshot := screenshot.all_after(output_path + os.path_separator) @@ -204,16 +223,33 @@ fn compare_screenshots(opt Options, base_path string, output_path string, target eprintln('Comparing `$src` with `$target`') } - result := os.execute('$idiff_exe "$src" "$target"') - if opt.verbose { + diff_file := os.join_path(os.temp_dir(), os.file_name(src).all_before_last('.') + + '.diff.tif') + diff_cmd := '$idiff_exe -p -fail 0.001 -failpercent 0.2 -od -o "$diff_file" -abs "$src" "$target"' + result := os.execute(diff_cmd) + if opt.verbose && result.exit_code == 0 { + eprintln('Running: $diff_cmd') eprintln('$result.output') } if result.exit_code != 0 { - fails[src] = target + eprintln('$result.output') + if result.exit_code == 1 { + warns[src] = target + } else { + fails[src] = target + } } } + if warns.len > 0 { + eprintln('--- WARNINGS ---') + eprintln('The following files had warnings when compared to their targets') + for warn_src, warn_target in warns { + eprintln('$warn_src ~= $warn_target') + } + } if fails.len > 0 { + eprintln('--- ERRORS ---') eprintln('The following files did not match their targets') for fail_src, fail_target in fails { eprintln('$fail_src != $fail_target') @@ -222,6 +258,12 @@ fn compare_screenshots(opt Options, base_path string, output_path string, target fail_copy := os.join_path(os.temp_dir(), 'fail.' + first.all_after_last('.')) os.cp(first, fail_copy) or { panic(err) } eprintln('First failed file `$first` is copied to `$fail_copy`') + + diff_file := os.join_path(os.temp_dir(), os.file_name(first).all_before_last('.') + + '.diff.tif') + diff_copy := os.join_path(os.temp_dir(), 'diff.tif') + os.cp(diff_file, diff_copy) or { panic(err) } + eprintln('First failed diff file `$diff_file` is copied to `$diff_copy`') exit(1) } } @@ -242,7 +284,9 @@ fn take_screenshots(opt Options, app string, out_path string) ?[]string { mut screenshots := []string{} shots := os.ls(out_path) or { return error('Failed listing dir `$out_path`') } for shot in shots { - screenshots << os.join_path(out_path, shot) + if shot.starts_with(os.file_name(app).all_before_last('.')) { + screenshots << os.join_path(out_path, shot) + } } return screenshots }