Add ability to ask external caches to invalidate.

Add a cached_urls method to the Comment, FoiAttachment, InfoRequest,
PublicBody and User models, containing the entries that those models
wish to invalidate when an event is logged via log_event or when the
censor rules change. Entries are either a fixed path to be purged or
a regex beginning with '^' to be banned.

Adds a NotifyCacheJob that for each of the object's cached_urls, for
each locale, for each host in VARNISH_HOSTS, makes BAN or PURGE HTTP
requests to the host, to invalidate the relevant entries.

app/jobs/notify_cache_job.rb

@@ -0,0 +1,55 @@
+require 'net/http'
+
+class Net::HTTP::Purge < Net::HTTP::Get
+  METHOD = 'PURGE'
+end
+
+class Net::HTTP::Ban < Net::HTTP::Get
+  METHOD = 'BAN'
+end
+
+##
+# Job to notify a cache of URLs to be purged or banned, given an object
+# (that must have a cached_urls method).
+#
+# Examples:
+#   NotifyCacheJob.perform(InfoRequest.first)
+#   NotifyCacheJob.perform(FoiAttachment.first)
+#   NotifyCacheJob.perform(Comment.first)
+#
+class NotifyCacheJob < ApplicationJob
+  queue_as :default
+
+  def perform(object)
+    urls = object.cached_urls
+    locales = [''] + AlaveteliLocalization.available_locales.map { |locale|
+      '/' + locale
+    }
+    hosts = AlaveteliConfiguration.varnish_hosts
+    locales.each do |locale|
+      hosts.each do |host|
+        Net::HTTP.start(AlaveteliConfiguration.domain, 80, host, 6081) do |http|
+          urls.each do |url|
+            if url.start_with? '^'
+              request = Net::HTTP::Ban.new('/')
+              request['X-Invalidate-Pattern'] = '^' + locale + url[1..-1]
+            else
+              request = Net::HTTP::Purge.new(locale + url)
+            end
+            response = http.request(request)
+            result = response.code
+            if result == "200"
+              Rails.logger.debug(
+                "PURGE: Purged URL #{url} at #{host}: #{result}"
+              )
+            else
+              Rails.logger.warn(
+                "PURGE: Unable to purge URL #{url} at #{host}: status #{result}"
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

app/models/censor_rule.rb

@@ -75,6 +75,9 @@ def is_global?
   def expire_requests
     if info_request
       InfoRequestExpireJob.perform_later(info_request)
+      if AlaveteliConfiguration.varnish_hosts.present?
+        NotifyCacheJob.perform_later(info_request)
+      end
     elsif user
       InfoRequestExpireJob.perform_later(user, :info_requests)
     elsif public_body

app/models/comment.rb

@@ -193,6 +193,13 @@ def hide(editor:)
     end
   end
 
+  def cached_urls
+    [
+      request_path(info_request),
+      show_user_wall_path(url_name: user.url_name)
+    ]
+  end
+
   private
 
   def check_body_has_content

app/models/foi_attachment.rb

@@ -29,6 +29,8 @@
 require 'digest'
 
 class FoiAttachment < ApplicationRecord
+  include Rails.application.routes.url_helpers
+  include LinkToHelper
   include MessageProminence
 
   MissingAttachment = Class.new(StandardError)
@@ -318,6 +320,12 @@ def body_as_html(dir, opts = {})
     AttachmentToHTML.to_html(self, to_html_opts)
   end
 
+  def cached_urls
+    [
+      request_path(incoming_message.info_request)
+    ]
+  end
+
   private
 
   def text_type?

app/models/info_request.rb

@@ -52,6 +52,7 @@ class InfoRequest < ApplicationRecord
   include InfoRequest::TitleValidation
   include Taggable
   include Notable
+  include LinkToHelper
 
   admin_columns exclude: %i[title url_title],
                 include: %i[rejected_incoming_count]
@@ -1163,6 +1164,15 @@ def log_event(type, params, options = {})
     if !last_event_time || (event.created_at > last_event_time)
       update_column(:last_event_time, event.created_at)
     end
+    if AlaveteliConfiguration.varnish_hosts.present?
+      if %w[comment edit_comment hide_comment].include? type
+        NotifyCacheJob.perform_later(Comment.find(params[:comment_id]))
+      elsif type == 'edit_attachment'
+        NotifyCacheJob.perform_later(FoiAttachment.find(params[:attachment_id]))
+      else
+        NotifyCacheJob.perform_later(self)
+      end
+    end
     event
   end
 
@@ -1758,6 +1768,34 @@ def latest_refusals
     incoming_messages.select(&:refusals?).last&.refusals || []
   end
 
+  def cached_urls
+    feed_request = TrackThing.new(
+      info_request: self,
+      track_type: 'request_updates'
+    )
+    feed_body = TrackThing.new(
+      public_body: public_body,
+      track_type: 'public_body_updates'
+    )
+    feed_user = TrackThing.new(
+      tracked_user: user,
+      track_type: 'user_updates'
+    )
+    [
+      '/',
+      public_body_path(public_body),
+      request_path(self),
+      request_details_path(self),
+      '^/list',
+      do_track_path(feed_request, feed = 'feed'),
+      '^/feed/list/',
+      do_track_path(feed_body, feed = 'feed'),
+      do_track_path(feed_user, feed = 'feed'),
+      user_path(user),
+      show_user_wall_path(url_name: user.url_name)
+    ]
+  end
+
   private
 
   def self.add_conditions_from_extra_params(params, extra_params)

app/models/public_body.rb

@@ -35,6 +35,8 @@
 class PublicBody < ApplicationRecord
   include Taggable
   include Notable
+  include Rails.application.routes.url_helpers
+  include LinkToHelper
 
   class ImportCSVDryRun < StandardError; end
 
@@ -114,7 +116,7 @@ def self.admin_title
 
   after_save :update_missing_email_tag
 
-  after_update :reindex_requested_from
+  after_update :reindex_requested_from, :invalidate_cached_pages
 
   # Every public body except for the internal admin one is visible
   scope :visible, -> { where("public_bodies.id <> #{ PublicBody.internal_admin_body.id }") }
@@ -911,6 +913,14 @@ def questions
     PublicBodyQuestion.fetch(self)
   end
 
+  def cached_urls
+    [
+      public_body_path(self),
+      list_public_bodies_path,
+      '^/body/list'
+    ]
+  end
+
   private
 
   # If the url_name has changed, then all requested_from: queries will break
@@ -919,6 +929,11 @@ def reindex_requested_from
     expire_requests if saved_change_to_attribute?(:url_name)
   end
 
+  def invalidate_cached_pages
+    return unless AlaveteliConfiguration.varnish_hosts.present?
+    NotifyCacheJob.perform_later(self)
+  end
+
   # Read an attribute value (without using locale fallbacks if the
   # attribute is translated)
   def read_attribute_value(name, locale)

app/models/user.rb

@@ -47,6 +47,8 @@ class User < ApplicationRecord
   include User::OneTimePassword
   include User::Slug
   include User::Survey
+  include Rails.application.routes.url_helpers
+  include LinkToHelper
 
   DEFAULT_CONTENT_LIMITS = {
     info_requests: AlaveteliConfiguration.max_requests_per_user_per_day,
@@ -179,7 +181,9 @@ class User < ApplicationRecord
   validate :email_and_name_are_valid
 
   after_initialize :set_defaults
-  after_update :reindex_referencing_models, :update_pro_account
+  after_update :reindex_referencing_models,
+               :update_pro_account,
+               :invalidate_cached_pages
 
   acts_as_xapian texts: [:name, :about_me],
                  values: [
@@ -341,6 +345,11 @@ def expire_comments
     comments.find_each(&:reindex_request_events)
   end
 
+  def invalidate_cached_pages
+    return unless AlaveteliConfiguration.varnish_hosts.present?
+    NotifyCacheJob.perform_later(self)
+  end
+
   def locale
     (super || AlaveteliLocalization.locale).to_s
   end
@@ -673,6 +682,12 @@ def flipper_id
     "User;#{id}"
   end
 
+  def cached_urls
+    [
+      user_path(self)
+    ]
+  end
+
   private
 
   def set_defaults

config/general.yml-example

@@ -627,6 +627,20 @@ EXCEPTION_NOTIFICATIONS_TO:
 # ---
 MAX_REQUESTS_PER_USER_PER_DAY: 6
 
+# If you're running behind Varnish set this to work out where to send purge
+# requests. Otherwise, don't set it.
+#
+# VARNISH_HOSTS - Array of Strings (default: nil)
+#
+# Examples:
+#
+#   VARNISH_HOSTS:
+#     - host1
+#     - host2
+#
+# ---
+VARNISH_HOSTS: null
+
 # Adding a value here will enable Google Analytics on all non-admin pages for
 # non-admin users.
 #

lib/configuration.rb

@@ -131,6 +131,7 @@ module AlaveteliConfiguration
       USE_MAILCATCHER_IN_DEVELOPMENT: true,
       USER_SIGN_IN_ACTIVITY_RETENTION_DAYS: 0,
       UTILITY_SEARCH_PATH: ['/usr/bin', '/usr/local/bin'],
+      VARNISH_HOSTS: [],
       WORKING_OR_CALENDAR_DAYS: 'working'
     }
     # rubocop:enable Layout/LineLength

spec/controllers/admin/foi_attachments_controller_spec.rb

@@ -78,6 +78,9 @@
       end
 
       it 'should log an "edit_attachment" event on the info_request' do
+        allow(AlaveteliConfiguration).to receive(:varnish_hosts).
+          and_return(['varnish'])
+        expect(NotifyCacheJob).to receive(:perform_later).with(attachment)
         allow(@controller).to receive(:admin_current_user).
           and_return("Admin user")
 

spec/controllers/admin_incoming_message_controller_spec.rb

@@ -17,6 +17,9 @@
     end
 
     it "destroys the ActiveStorage attachment record" do
+      allow(AlaveteliConfiguration).to receive(:varnish_hosts).
+        and_return(['varnish'])
+      expect(NotifyCacheJob).to receive(:perform_later).with(@im.info_request)
       file = @im.raw_email.file
       expect(file.attached?).to eq true
       post :destroy, params: { id: @im.id }
@@ -217,6 +220,10 @@ def make_request(params=@default_params)
 
     it 'should log an "edit_incoming" event on the info_request' do
       allow(@controller).to receive(:admin_current_user).and_return("Admin user")
+      allow(AlaveteliConfiguration).to receive(:varnish_hosts).
+        and_return(['varnish'])
+      expect(NotifyCacheJob).to receive(:perform_later).
+        with(@incoming.info_request)
       make_request
       @incoming.reload
       last_event = @incoming.info_request_events.last

spec/jobs/notify_cache_job_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+
+RSpec.describe NotifyCacheJob, type: :job do
+  let(:args) { [] }
+  subject(:perform) { described_class.new.perform(*args) }
+
+  let(:request) { FactoryBot.build(:info_request) }
+  let(:comment) { FactoryBot.build(:comment) }
+  let(:attachment) { FactoryBot.build(:pdf_attachment) }
+  let(:im) { FactoryBot.create(:plain_incoming_message) }
+
+  before do
+    @old_include_default_locale_in_urls =
+      AlaveteliConfiguration.include_default_locale_in_urls
+    AlaveteliLocalization.set_default_locale_urls(false)
+
+    allow(AlaveteliConfiguration).to receive(:varnish_hosts).
+      and_return(['varnish'])
+
+    stub_request(:purge, /^http:\/\/test\.host(\/(en|es|fr|en_GB))?\/(|((feed\/)?body|(feed\/|details\/)?request|(feed\/)?user)\/[a-z0-9_]+|user\/[a-z0-9_]+\/wall)$/).
+      to_return(status: 200, body: "", headers: {})
+    stub_request(:ban, 'http://test.host/').
+      with(headers:
+        {
+          'X-Invalidate-Pattern' =>
+            /^\^(\/(en|es|fr|en_GB))?\/(list|feed\/list\/)$/
+        }).
+      to_return(status: 200, body: "", headers: {})
+  end
+
+  after do
+    AlaveteliLocalization.set_default_locale_urls(
+      @old_include_default_locale_in_urls
+    )
+  end
+
+  context 'when called with a request' do
+    let(:args) { [request] }
+    it 'calls out to varnish correctly' do
+      expect(Rails.logger).to receive(:debug).exactly(55).times
+      perform
+    end
+  end
+
+  context 'when called with a comment' do
+    let(:args) { [comment] }
+    it 'calls out to varnish correctly' do
+      expect(Rails.logger).to receive(:debug).exactly(10).times
+      perform
+    end
+  end
+
+  context 'when called with an attachment' do
+    let(:args) { [attachment] }
+    it 'calls out to varnish correctly' do
+      attachment.incoming_message = im
+      expect(Rails.logger).to receive(:debug).exactly(5).times
+      perform
+    end
+  end
+end

spec/models/censor_rule_spec.rb

@@ -159,12 +159,15 @@
   end
 
   describe '#expire_requests' do
-
     it 'create expire job for the request if it is a request rule' do
+      allow(AlaveteliConfiguration).to receive(:varnish_hosts).
+        and_return(['varnish'])
+
       request = FactoryBot.create(:info_request)
       rule = FactoryBot.create(:info_request_censor_rule,
                                info_request: request)
       expect(InfoRequestExpireJob).to receive(:perform_later).with(request)
+      expect(NotifyCacheJob).to receive(:perform_later).with(request)
       rule.expire_requests
     end