From ecc15c17f6f5b3ca8125b67359359fea2eb4c8f6 Mon Sep 17 00:00:00 2001 From: David Singleton Date: Fri, 29 Jan 2016 17:14:27 +0000 Subject: [PATCH] Add wraith for visual regression testing Add Wraith so that regressions can be easily spotted. Runs against a set of supported examples. Run while pointing government-frontend at the dummy content store, and an instance of static. Usage: On master run: wraith history test/wraith/config.yaml On local branch run: wraith latest test/wraith/config.yaml Had to run `bundle update` to get a version of nokogiri that would work with `wraith`. Configuration generated with `wraith setup` and combined with our existing wraith usage, [1]. With modern versions of wraith you don't need a `snap.js` file, as wraith will take care of the `phantomjs` intergration, however we still need to set some cookies to prevent intermitent UI (like the survey, or cookie bar) from interfering with the diff results. [1] https://github.com/alphagov/design-principles/pull/246 This config seems to work OK-ish, though starts having time out issues with many combinations of screen_widths, dogpiling the rails app and getting timeouts. I couldn't find an obvious way to limit this without forking wraith, so for now i've used a smaller set of screen_widths. I also had to specifcy explicit heights for each screen_width, this seems to be new to Wraith 3.x. Previous versions would capture the full page as a default, where as now it will default to 1500px unless you specificy a height. This isn't ideal as most of the page on a small screen_size is much further down than 1500px. For now i've set some heights that deal with some common cases, but i'm not sure this is a good approach long term. There are some related wraith notes on that here: - https://github.com/BBC-News/wraith/issues/318 (suggestion doesn't work) - https://github.com/BBC-News/wraith/issues/296 --- Gemfile | 5 +- Gemfile.lock | 62 +++++++++++++++--------- README.md | 19 ++++++++ test/wraith/config.yaml | 76 ++++++++++++++++++++++++++++++ test/wraith/cookies_and_headers.js | 21 +++++++++ 5 files changed, 160 insertions(+), 23 deletions(-) create mode 100644 test/wraith/config.yaml create mode 100644 test/wraith/cookies_and_headers.js diff --git a/Gemfile b/Gemfile index 81e42b209..8eb7d0e89 100644 --- a/Gemfile +++ b/Gemfile @@ -20,7 +20,10 @@ else gem 'gds-api-adapters', '27.0.0' end -gem 'govuk-lint', group: [:development, :test] +group :development, :test do + gem 'govuk-lint' + gem 'wraith', '~> 3.1' +end group :development do gem 'better_errors' diff --git a/Gemfile.lock b/Gemfile.lock index 2a0839568..6bf49db47 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -32,6 +32,9 @@ GEM airbrake (4.0.0) builder multi_json + anemone (0.7.2) + nokogiri (>= 1.3.0) + robotex (>= 1.0.0) arel (5.0.1.20140414130214) ast (2.2.0) astrolabe (1.3.1) @@ -43,23 +46,23 @@ GEM binding_of_caller (0.7.2) debug_inspector (>= 0.0.1) builder (3.2.2) - byebug (5.0.0) - columnize (= 0.9.0) - capybara (2.5.0) + byebug (8.2.1) + capybara (2.6.2) + addressable mime-types (>= 1.16) nokogiri (>= 1.3.3) rack (>= 1.0.0) rack-test (>= 0.5.4) xpath (~> 2.0) + casperjs (1.0.0) coderay (1.1.0) - columnize (0.9.0) - crack (0.4.2) + crack (0.4.3) safe_yaml (~> 1.0.0) debug_inspector (0.0.2) - domain_name (0.5.24) + domain_name (0.5.20160128) unf (>= 0.0.5, < 1.0.0) erubis (2.7.0) - execjs (2.5.2) + execjs (2.6.0) gds-api-adapters (27.0.0) link_header lrucache (~> 0.1.1) @@ -78,11 +81,13 @@ GEM http-cookie (1.0.2) domain_name (~> 0.5) i18n (0.7.0) + image_size (1.4.1) json (1.8.3) - json-schema (2.5.1) - addressable (~> 2.3.7) - kgio (2.9.3) + json-schema (2.5.2) + addressable (~> 2.3.8) + kgio (2.10.0) link_header (0.0.8) + log4r (1.1.10) logstash-event (1.1.5) logstasher (0.6.1) logstash-event (~> 1.1.0) @@ -93,13 +98,14 @@ GEM mime-types (>= 1.16, < 3) method_source (0.8.2) mime-types (2.99) - mini_portile (0.6.2) + mini_portile2 (2.0.0) minitest (5.8.4) multi_json (1.11.2) - netrc (0.10.3) - nokogiri (1.6.6.2) - mini_portile (~> 0.6.0) + netrc (0.11.0) + nokogiri (1.6.7) + mini_portile2 (~> 2.0.0.rc2) null_logger (0.0.1) + parallel (1.6.1) parser (2.3.0.2) ast (~> 2.2) plek (1.11.0) @@ -108,11 +114,11 @@ GEM coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) - pry-byebug (3.2.0) - byebug (~> 5.0) + pry-byebug (3.3.0) + byebug (~> 8.0) pry (~> 0.10) rack (1.5.5) - rack-cache (1.5.1) + rack-cache (1.6.0) rack (>= 0.4) rack-test (0.6.3) rack (>= 1.0) @@ -126,8 +132,8 @@ GEM bundler (>= 1.3.0, < 2.0) railties (= 4.1.14.1) sprockets-rails (~> 2.0) - rails-i18n (4.0.4) - i18n (~> 0.6) + rails-i18n (4.0.8) + i18n (~> 0.7) railties (~> 4.0) rails_translation_manager (0.0.2) activesupport @@ -138,13 +144,14 @@ GEM rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) rainbow (2.1.0) - raindrops (0.14.0) + raindrops (0.15.0) rake (10.5.0) - request_store (1.1.0) + request_store (1.3.0) rest-client (1.8.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 3.0) netrc (~> 0.7) + robotex (1.0.0) rubocop (0.35.1) astrolabe (~> 1.3) parser (>= 2.2.3.0, < 3.0) @@ -184,7 +191,7 @@ GEM tins (1.6.0) tzinfo (1.2.2) thread_safe (~> 0.1) - uglifier (2.7.1) + uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) unf (0.1.4) @@ -197,6 +204,16 @@ GEM webmock (1.18.0) addressable (>= 2.3.6) crack (>= 0.3.2) + wraith (3.1.0) + anemone + casperjs + image_size + log4r + nokogiri (= 1.6.7) + parallel + rake + robotex + thor xpath (2.0.0) nokogiri (~> 1.3) @@ -223,3 +240,4 @@ DEPENDENCIES uglifier (>= 1.3.0) unicorn (= 4.8) webmock (~> 1.18.0) + wraith (~> 3.1) diff --git a/README.md b/README.md index 3ab0cb7fe..bf9bd6a01 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,25 @@ Or to specify the location explicitly: `GOVUK_CONTENT_SCHEMAS_PATH=/some/dir/govuk-content-schemas bundle exec rake` +#### Visual regression tests + +Use [Wraith](http://bbc-news.github.io/wraith/) ("A responsive screenshot +comparison tool") to generate a visual diff to compare rendering changes in this +application. Wraith has some dependencies you'll [need to install](http://bbc-news.github.io/wraith/os-install.html). + +First, on `master` branch, run: +``` +wraith history test/wraith/config.yaml +``` + +Then, on a branch with your changes, run: +``` +wraith latest test/wraith/config.yaml +``` + +This will generate image diffs comparing the two runs, including a browseable +gallery of the output, in `tmp/wraith`. + ## Licence [MIT License](LICENCE) diff --git a/test/wraith/config.yaml b/test/wraith/config.yaml new file mode 100644 index 000000000..b3e2c81cc --- /dev/null +++ b/test/wraith/config.yaml @@ -0,0 +1,76 @@ +############################################################## +############################################################## +# This is an example configuration provided by Wraith. +# Feel free to amend for your own requirements. +# --- +# This particular config is intended to demonstrate how +# to use Wraith in 'history' mode, which is best suited to +# making sure your site's appearance remains consistent over +# time. +# +# `wraith history history.yaml` # generate base screenshots +# `wraith latest history.yaml` # take new shots and compare +# --- +# Installing: +# +# On a GOV.UK VM, run: +# `apt-get install imagemagick phantomjs` +# +# Otherwise, on a Mac, run: +# `brew install imagemagick phantomjs` +# +############################################################## +############################################################## + +# (required) The engine to run Wraith with. Examples: 'phantomjs', 'casperjs', 'slimerjs' +browser: + phantomjs: 'phantomjs' + +# (optional) JavaScript file to execute before taking screenshot of every path. Default: nil +before_capture: 'test/wraith/cookies_and_headers.js' + +# (required) The directory that your latest screenshots will be stored in +directory: 'tmp/wraith/shots' + +# (required for history mode, otherwise optional) The directory that your base screenshots will be stored in. +history_dir: 'tmp/wraith/shots_history' + +# (required) Amount of fuzz ImageMagick will use when comparing images. A higher fuzz makes the comparison less strict. +fuzz: '20%' + +# (required) The domain to take screenshots of. +domains: + dev: 'http://localhost:3090' + +# (required) Screen widths (and optional height) to resize the browser to before taking the screenshot. +screen_widths: + - 320x5000 + - 600x4000 + - 1080x3000 + +# (optional) Resize to each screen width (efficient), or reload at each screen width (costly). Default: 'reload' +resize_or_reload: 'reload' + +# (optional) Color to highlight the image diff. Default: 'blue' +highlight_color: 'red' + +# (optional) Choose which results are displayed in the gallery, and in what order. Default: alphanumeric +# Options: +# alphanumeric - all paths (with or without a difference) are shown, sorted by path +# diffs_first - all paths (with or without a difference) are shown, sorted by difference size (largest first) +# diffs_only - only paths with a difference are shown, sorted by difference size (largest first) +# Note: different screen widths are always grouped together. +mode: 'diffs_first' + +# (optional) The maximum acceptable level of difference (in %) between two images before Wraith reports a failure. Default: 0 +threshold: 5 + +# (required) The paths to capture. +paths: + case_study: '/government/case-studies/doing-business-in-spain.es' + case_study_withdrawn: '/government/case-studies/terence-age-33-stoke-on-trent' + take_part: '/government/get-involved/take-part/become-a-councillor' + statistics_announcement: '/government/statistics/announcements/diagnostic-imaging-dataset-for-september-2015--2' + statistics_national: '/government/statistics/announcements/uk-armed-forces-quarterly-personnel-report-october-2015' + statistics_announcement_release_date_changes: '/government/statistics/announcements/diagnostic-imaging-dataset-for-september-2015--1' + statistics_announcement_cancelled: '/government/statistics/announcements/diagnostic-imaging-dataset-for-september-2015' diff --git a/test/wraith/cookies_and_headers.js b/test/wraith/cookies_and_headers.js new file mode 100644 index 000000000..55cd8d1bc --- /dev/null +++ b/test/wraith/cookies_and_headers.js @@ -0,0 +1,21 @@ +// ###################################################### +// This is an example module provided by Wraith. +// Feel free to amend for your own requirements. +// ###################################################### +module.exports = function (phantom, ready) { + phantom.addCookie({ + 'name': 'govuk_takenUserSatisfactionSurvey', + 'value': 'yes', + 'domain': 'localhost' + }); + + phantom.addCookie({ + 'name': 'seen_cookie_message', + 'value': 'yes', + 'domain': 'localhost' + }); + + phantom.open(phantom.url, function () { + setTimeout(ready, 1000); + }); +}