Minimum viable component guide visual regression test

.gitignore

@@ -1,3 +1,4 @@
 results/visual
 config/tmp*
+config/spider_paths*
 node_modules/

Gemfile

@@ -1,5 +1,7 @@
 source "https://rubygems.org"
+ruby File.read(".ruby-version").chomp
 
+gem "sinatra"
 gem "wraith", "~> 4.0"
 gem "govuk-lint", "~> 0.8"
 gem "rake", "~> 10.0"
@@ -8,7 +10,7 @@ gem "json"
 
 # Yarn is only available to Ruby apps on Heroku with the webpacker gem
 # https://devcenter.heroku.com/changelog-items/1114
-gem "webpacker"
+gem "webpacker", require: false
 
 group :development, :test do
   gem "rspec", "~> 3.6"

Gemfile.lock

@@ -50,6 +50,7 @@ GEM
     mime-types-data (3.2016.0521)
     mini_portile2 (2.2.0)
     minitest (5.10.3)
+    mustermann (1.0.1)
     netrc (0.11.0)
     nokogiri (1.8.0)
       mini_portile2 (~> 2.2.0)
@@ -65,6 +66,8 @@ GEM
       byebug (~> 9.1)
       pry (~> 0.10)
     rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
     rack-proxy (0.6.2)
       rack
     rack-test (0.7.0)
@@ -113,9 +116,15 @@ GEM
     scss_lint (0.44.0)
       rake (~> 10.0)
       sass (~> 3.4.15)
+    sinatra (2.0.0)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.0)
+      tilt (~> 2.0)
     slop (3.6.0)
     thor (0.20.0)
     thread_safe (0.3.6)
+    tilt (2.0.8)
     tins (1.6.0)
     tzinfo (1.2.3)
       thread_safe (~> 0.1)
@@ -145,8 +154,12 @@ DEPENDENCIES
   rake (~> 10.0)
   rest-client
   rspec (~> 3.6)
+  sinatra
   webpacker
   wraith (~> 4.0)
 
+RUBY VERSION
+   ruby 2.3.1p112
+
 BUNDLED WITH
    1.15.1

Procfile

@@ -0,0 +1 @@
+web: bundle exec ruby app.rb

app.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'sinatra'
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'govuk_visual_regression'
+set :bind, '0.0.0.0'
+
+post '/run' do
+  begin
+    payload = JSON.parse(request.body.read)
+  rescue JSON::ParserError
+    status 400
+  end
+
+  environment = payload['deployment']['environment']
+  review_domain = "https://#{environment}.herokuapp.com"
+  live_domain = "https://#{environment.gsub(/\-pr\-\d+$/, '')}.herokuapp.com"
+  surge_domain = "#{environment}.surge.sh"
+
+  GovukVisualRegression::VisualDiff::Runner.new(review_domain: review_domain).spider_component_guide
+  paths = GovukVisualRegression::VisualDiff::SpiderPaths.component_preview_paths
+  GovukVisualRegression::VisualDiff::HerokuRunner.new(paths: paths, review_domain: review_domain, live_domain: live_domain, surge_domain: surge_domain).run
+end

config/wraith.yaml

@@ -5,7 +5,6 @@ browser:
 directory: results/visual
 screen_widths:
   - 320x5000
-  - 600x4000
   - 1080x3000
 resize_or_reload: 'resize'
 mode: diffs_first
@@ -20,9 +19,10 @@ gallery:
 # These paths will be populated by GovukVisualRegression::VisualDiff::Runner
 paths:
 
+# Domains can be overwritten by GovukVisualRegression::VisualDiff::Runner
 domains:
-  production: https://www.gov.uk
-  master: https://government-frontend.herokuapp.com
+  live: https://www.gov.uk
+  review: https://government-frontend.herokuapp.com
 
 # Use TLS v1 when requesting https://www.gov.uk/… from within the VM
 # Ignore SSL errors when accessing staging

lib/govuk_visual_regression.rb

@@ -13,6 +13,10 @@ def self.wraith_config_template
     config_file 'wraith.yaml'
   end
 
+  def self.spider_paths
+    config_file 'spider_paths.yml'
+  end
+
   def self.config_file(filename)
     File.expand_path(root_dir + "/config/#{filename}")
   end

lib/govuk_visual_regression/tasks/rakefile.rake

@@ -6,6 +6,14 @@ def load_paths
   YAML.load(open(ENV.fetch("URI")))
 end
 
+def review_domain
+  ENV.fetch("REVIEW_DOMAIN")
+end
+
+def live_domain
+  ENV.fetch("LIVE_DOMAIN")
+end
+
 namespace :diff do
   desc 'Set env var `URI` with location of a yaml file containing paths to diff'
   task visual: ['config:pre_flight_check'] do |_t, args|
@@ -20,6 +28,16 @@ namespace :diff do
     GovukVisualRegression::VisualDiff::Runner.new(paths: paths).run
   end
 
+  desc 'Set env var `REVIEW_DOMAIN` to the domain hosting the newer component guide and `LIVE_DOMAIN` to compare with'
+  task component_guide: ['spider:component_guide'] do |_t, args|
+    paths = GovukVisualRegression::VisualDiff::SpiderPaths.component_preview_paths
+    if paths.any?
+      GovukVisualRegression::VisualDiff::Runner.new(paths: paths, review_domain: review_domain, live_domain: live_domain).run
+    else
+      puts "No paths found"
+    end
+  end
+
   desc "clears the results directory"
   task :clear_results do
     puts "---> Clearing results directory"
@@ -28,6 +46,13 @@ namespace :diff do
   end
 end
 
+namespace :spider do
+  desc 'Set env var `REVIEW_DOMAIN` to the domain hosting the component guide'
+  task component_guide: ['config:pre_flight_check'] do |_t, args|
+    GovukVisualRegression::VisualDiff::Runner.new(review_domain: review_domain).spider_component_guide
+  end
+end
+
 namespace :config do
   desc "Checks that dependencies are in place"
   task :pre_flight_check do

lib/govuk_visual_regression/visual_diff.rb

@@ -3,6 +3,8 @@ module VisualDiff
     autoload :Runner, 'govuk_visual_regression/visual_diff/runner'
     autoload :HerokuRunner, 'govuk_visual_regression/visual_diff/heroku_runner'
     autoload :WraithConfig, 'govuk_visual_regression/visual_diff/wraith_config'
+    autoload :WraithSpiderComponentGuideConfig, 'govuk_visual_regression/visual_diff/wraith_spider_component_guide_config'
     autoload :DocumentTypes, 'govuk_visual_regression/visual_diff/document_types'
+    autoload :SpiderPaths, 'govuk_visual_regression/visual_diff/spider_paths'
   end
 end

lib/govuk_visual_regression/visual_diff/heroku_runner.rb

@@ -1,12 +1,20 @@
 module GovukVisualRegression
   module VisualDiff
     class HerokuRunner < Runner
+      def initialize(paths: [], review_domain: nil, live_domain: nil, surge_domain: nil, kernel: Kernel)
+        @paths = paths
+        @kernel = kernel
+        @review_domain = review_domain
+        @live_domain = live_domain
+        @surge_domain = surge_domain
+      end
+
       def install_surge
         @kernel.system "yarn global add surge"
       end
 
       def upload_to_surge
-        @kernel.system "surge --project results/visual/ --domain govuk-vr.surge.sh"
+        @kernel.system "surge --project results/visual/ --domain #{@surge_domain}"
       end
 
       def run

lib/govuk_visual_regression/visual_diff/runner.rb

@@ -1,13 +1,25 @@
 module GovukVisualRegression
   module VisualDiff
     class Runner
-      def initialize(paths:, kernel: Kernel)
+      def initialize(paths: [], review_domain: nil, live_domain: nil, kernel: Kernel)
         @paths = paths
         @kernel = kernel
+        @review_domain = review_domain
+        @live_domain = live_domain
+      end
+
+      def spider_component_guide
+        wraith_config = WraithSpiderComponentGuideConfig.new(review_domain: @review_domain)
+        wraith_config.write
+
+        cmd = "wraith spider #{wraith_config.location}"
+        puts "---> Running component guide spider on #{@review_domain}"
+        puts "running: #{cmd}"
+        @kernel.system cmd
       end
 
       def run
-        wraith_config = WraithConfig.new(paths: @paths)
+        wraith_config = WraithConfig.new(paths: @paths, review_domain: @review_domain, live_domain: @live_domain)
         wraith_config.write
 
         cmd = "wraith capture #{wraith_config.location}"

lib/govuk_visual_regression/visual_diff/spider_paths.rb

@@ -0,0 +1,18 @@
+require 'yaml'
+
+module GovukVisualRegression
+  module VisualDiff
+    module SpiderPaths
+      def self.paths
+        YAML.load_file(GovukVisualRegression.spider_paths)['paths']
+      end
+
+      # Keep number of pages test down to a minimum
+      # Only test component previews, not the guide itself
+      # And only test the pages that show all previews on one page
+      def self.component_preview_paths
+        paths.values.select { |v| v.match(/preview$/) && v.split('/').length == 4 }
+      end
+    end
+  end
+end

lib/govuk_visual_regression/visual_diff/wraith_config.rb

@@ -1,29 +1,58 @@
 require 'yaml'
-require 'securerandom'
 
 module GovukVisualRegression
   module VisualDiff
     class WraithConfig
       attr_reader :location
+      attr_reader :paths
+      attr_reader :review_domain
+      attr_reader :live_domain
 
-      def initialize(paths:)
+      def initialize(paths: [], review_domain: nil, live_domain: nil)
         @paths = paths
-        @location = GovukVisualRegression.config_file("tmp_wraith_config.yaml")
+        @live_domain = live_domain
+        @review_domain = review_domain
+        @location = GovukVisualRegression.config_file(temporary_config_file_name)
       end
 
-      def write
-        config_template = YAML.load_file GovukVisualRegression.wraith_config_template
-        wraith_formatted_paths = @paths.each_with_object({}) do |path, hash|
-          hash[SecureRandom.uuid] = path
+      def config
+        config = YAML.load_file(GovukVisualRegression.wraith_config_template)
+        config["paths"] = paths.each_with_object({}) do |path, hash|
+          hash[path_config_name(path)] = path unless path_would_break_wraith?(path)
         end
-        config_template["paths"] = wraith_formatted_paths
+
+        config["domains"]["live"] = live_domain if live_domain
+        config["domains"]["review"] = review_domain if review_domain
+        config
+      end
+
+      def yaml_config
+        YAML.dump(config)
+      end
+
+      def write
         wraith_config = File.new(location, "w")
-        wraith_config.write(YAML.dump config_template)
+        wraith_config.write(yaml_config)
         wraith_config.tap(&:close)
       end
 
       def delete
-        File.unlink location
+        File.unlink(location)
+      end
+
+      # TODO: Remove when Wraith patched
+      # Paths containing "path" in them break Wraith:
+      # https://github.com/BBC-News/wraith/issues/536
+      def path_would_break_wraith?(path)
+        path.include?('path')
+      end
+
+      def path_config_name(path)
+        path.gsub('/', '_')
+      end
+
+      def temporary_config_file_name
+        "tmp_wraith_config.yaml"
       end
     end
   end

lib/govuk_visual_regression/visual_diff/wraith_spider_component_guide_config.rb

@@ -0,0 +1,23 @@
+module GovukVisualRegression
+  module VisualDiff
+    class WraithSpiderComponentGuideConfig < WraithConfig
+      def config
+        config = YAML.load_file(GovukVisualRegression.wraith_config_template)
+        config["domains"].delete("live")
+        config["domains"]["review"] = "#{review_domain}/component-guide" if review_domain
+
+        # http://bbc-news.github.io/wraith/#Spiderfunctionality
+        config["imports"] = "spider_paths.yml"
+
+        # Skip pages that don't begin with /component-guide
+        config["spider_skips"] = [/^\/(?!component\-guide)/]
+        config.delete("paths")
+        config
+      end
+
+      def temporary_config_file_name
+        "tmp_wraith_spider_component_guide_config.yaml"
+      end
+    end
+  end
+end

spec/fixtures/deployment_payload.json

@@ -0,0 +1,5 @@
+{
+  "deployment": {
+    "environment": "government-frontend-pr-479"
+  }
+}

spec/govuk_visual_regression/visual_diff/runner_spec.rb

@@ -3,7 +3,7 @@
     let(:kernel) { double }
     let(:input_paths) { FixtureHelper.load_paths_from("test_paths.yaml") }
     let(:config_handler_klass) { GovukVisualRegression::VisualDiff::WraithConfig }
-    let(:config_handler) { config_handler_klass.new(paths: input_paths) }
+    let(:config_handler) { config_handler_klass.new(paths: input_paths, review_domain: 'review') }
 
     before do
       allow(config_handler_klass).to receive(:new).and_return(config_handler)
@@ -12,7 +12,7 @@
     end
 
     it "executes wraith with the appropriate config" do
-      expect(config_handler_klass).to receive(:new).with(paths: ["/government/stats/foo", "/government/stats/bar"])
+      expect(config_handler_klass).to receive(:new).with(paths: ["/government/stats/foo", "/government/stats/bar"], review_domain: nil, live_domain: nil)
       expect(config_handler).to receive(:write)
       expect(kernel).to receive(:system).with("wraith capture #{config_handler.location}")
       expect(config_handler).to receive(:delete)

spec/govuk_visual_regression/visual_diff/wraith_config_spec.rb

@@ -1,7 +1,7 @@
 describe GovukVisualRegression::VisualDiff::WraithConfig do
   describe "config write and cleanup" do
     it "writes out a wraith config with the specified paths, and cleans up after" do
-      wraith_config = described_class.new(paths: %w{foo bar })
+      wraith_config = described_class.new(paths: %w{foo bar }, review_domain: 'review')
       wraith_config.write
 
       expect(File.exist? wraith_config.location).to be true
@@ -11,5 +11,16 @@
       wraith_config.delete
       expect(File.exist? wraith_config.location).to be false
     end
+
+    it "ignores paths containing the word path" do
+      wraith_config = described_class.new(paths: %w{/foo /foo/bar /some/path }, review_domain: 'review')
+      wraith_config.write
+
+      config_data = YAML.load_file wraith_config.location
+      expect(config_data["paths"].values).to match_array %w{/foo /foo/bar}
+
+      wraith_config.delete
+      expect(File.exist? wraith_config.location).to be false
+    end
   end
 end