Add ability to ask external caches to invalidate.

Add a cached_urls method to the Comment, FoiAttachment, InfoRequest,
PublicBody and User models, containing the entries that those models
wish to invalidate either when an event is logged, via log_event, or
when the censor rules change, or when a body or user is updated. The
entries are fixed paths to be purged, or regexes beginning with ^ to
be banned.

Adds a NotifyCacheJob that for each of the object's cached_urls, for
each locale, for each host in VARNISH_HOSTS, makes BAN or PURGE HTTP
requests to the host, to invalidate the relevant entries.

app/jobs/notify_cache_job.rb

@@ -0,0 +1,68 @@
+require 'net/http'
+
+class Net::HTTP::Purge < Net::HTTP::Get
+  METHOD = 'PURGE'
+end
+
+class Net::HTTP::Ban < Net::HTTP::Get
+  METHOD = 'BAN'
+end
+
+##
+# Job to notify a cache of URLs to be purged or banned, given an object
+# (that must have a cached_urls method).
+#
+# Examples:
+#   NotifyCacheJob.perform(InfoRequest.first)
+#   NotifyCacheJob.perform(FoiAttachment.first)
+#   NotifyCacheJob.perform(Comment.first)
+#
+class NotifyCacheJob < ApplicationJob
+  queue_as :default
+
+  around_enqueue do |_, block|
+    block.call if AlaveteliConfiguration.varnish_hosts.present?
+  end
+
+  def perform(object)
+    urls = object.cached_urls
+    locales = [''] + AlaveteliLocalization.available_locales.map { "/#{_1}" }
+    hosts = AlaveteliConfiguration.varnish_hosts
+
+    urls.product(locales, hosts).each do |url, locale, host|
+      if url.start_with? '^'
+        request = Net::HTTP::Ban.new('/')
+        request['X-Invalidate-Pattern'] = '^' + locale + url[1..-1]
+      else
+        request = Net::HTTP::Purge.new(locale + url)
+      end
+
+      response = connection_for_host(host).request(request)
+      log_result = "#{request.method} #{url} at #{host}: #{response.code}"
+
+      case response
+      when Net::HTTPSuccess
+        Rails.logger.debug('NotifyCacheJob: ' + log_result)
+      else
+        Rails.logger.warn('NotifyCacheJob: Unable to ' + log_result)
+      end
+    end
+
+  ensure
+    close_connections
+  end
+
+  def connections
+    @connections ||= {}
+  end
+
+  def connection_for_host(host)
+    connections[host] ||= Net::HTTP.start(
+      AlaveteliConfiguration.domain, 80, host, 6081
+    )
+  end
+
+  def close_connections
+    connections.values.each { _1.finish if _1.started? }
+  end
+end

app/models/censor_rule.rb

@@ -75,6 +75,7 @@ def is_global?
   def expire_requests
     if info_request
       InfoRequestExpireJob.perform_later(info_request)
+      NotifyCacheJob.perform_later(info_request)
     elsif user
       InfoRequestExpireJob.perform_later(user, :info_requests)
     elsif public_body

app/models/comment.rb

@@ -193,6 +193,13 @@ def hide(editor:)
     end
   end
 
+  def cached_urls
+    [
+      request_path(info_request),
+      show_user_wall_path(url_name: user.url_name)
+    ]
+  end
+
   private
 
   def check_body_has_content

app/models/foi_attachment.rb

@@ -29,6 +29,8 @@
 require 'digest'
 
 class FoiAttachment < ApplicationRecord
+  include Rails.application.routes.url_helpers
+  include LinkToHelper
   include MessageProminence
 
   MissingAttachment = Class.new(StandardError)
@@ -318,6 +320,12 @@ def body_as_html(dir, opts = {})
     AttachmentToHTML.to_html(self, to_html_opts)
   end
 
+  def cached_urls
+    [
+      request_path(incoming_message.info_request)
+    ]
+  end
+
   private
 
   def text_type?

app/models/info_request.rb

@@ -52,6 +52,7 @@ class InfoRequest < ApplicationRecord
   include InfoRequest::TitleValidation
   include Taggable
   include Notable
+  include LinkToHelper
 
   admin_columns exclude: %i[title url_title],
                 include: %i[rejected_incoming_count]
@@ -1758,6 +1759,34 @@ def latest_refusals
     incoming_messages.select(&:refusals?).last&.refusals || []
   end
 
+  def cached_urls
+    feed_request = TrackThing.new(
+      info_request: self,
+      track_type: 'request_updates'
+    )
+    feed_body = TrackThing.new(
+      public_body: public_body,
+      track_type: 'public_body_updates'
+    )
+    feed_user = TrackThing.new(
+      tracked_user: user,
+      track_type: 'user_updates'
+    )
+    [
+      '/',
+      public_body_path(public_body),
+      request_path(self),
+      request_details_path(self),
+      '^/list',
+      do_track_path(feed_request, feed = 'feed'),
+      '^/feed/list/',
+      do_track_path(feed_body, feed = 'feed'),
+      do_track_path(feed_user, feed = 'feed'),
+      user_path(user),
+      show_user_wall_path(url_name: user.url_name)
+    ]
+  end
+
   private
 
   def self.add_conditions_from_extra_params(params, extra_params)

app/models/info_request_event.rb

@@ -88,6 +88,7 @@ class InfoRequestEvent < ApplicationRecord
     self.event_type = "hide"
   end
   after_create :update_request, if: :response?
+  after_create :invalidate_cached_pages
 
   after_commit -> { info_request.create_or_update_request_summary },
                   on: [:create]
@@ -355,6 +356,16 @@ def update_request
     info_request.update_last_public_response_at
   end
 
+  def invalidate_cached_pages
+    if comment
+      NotifyCacheJob.perform_later(comment)
+    elsif foi_attachment
+      NotifyCacheJob.perform_later(foi_attachment)
+    else
+      NotifyCacheJob.perform_later(info_request)
+    end
+  end
+
   def same_email_as_previous_send?
     prev_addr = info_request.get_previous_email_sent_to(self)
     curr_addr = params[:email]
@@ -406,6 +417,11 @@ def set_calculated_state!(state)
     end
   end
 
+  def foi_attachment
+    return unless params[:attachment_id]
+    @foi_attachment ||= FoiAttachment.find(params[:attachment_id])
+  end
+
   protected
 
   def variety

app/models/public_body.rb

@@ -35,6 +35,8 @@
 class PublicBody < ApplicationRecord
   include Taggable
   include Notable
+  include Rails.application.routes.url_helpers
+  include LinkToHelper
 
   class ImportCSVDryRun < StandardError; end
 
@@ -114,7 +116,7 @@ def self.admin_title
 
   after_save :update_missing_email_tag
 
-  after_update :reindex_requested_from
+  after_update :reindex_requested_from, :invalidate_cached_pages
 
   # Every public body except for the internal admin one is visible
   scope :visible, -> { where("public_bodies.id <> #{ PublicBody.internal_admin_body.id }") }
@@ -911,6 +913,14 @@ def questions
     PublicBodyQuestion.fetch(self)
   end
 
+  def cached_urls
+    [
+      public_body_path(self),
+      list_public_bodies_path,
+      '^/body/list'
+    ]
+  end
+
   private
 
   # If the url_name has changed, then all requested_from: queries will break
@@ -919,6 +929,10 @@ def reindex_requested_from
     expire_requests if saved_change_to_attribute?(:url_name)
   end
 
+  def invalidate_cached_pages
+    NotifyCacheJob.perform_later(self)
+  end
+
   # Read an attribute value (without using locale fallbacks if the
   # attribute is translated)
   def read_attribute_value(name, locale)

app/models/user.rb

@@ -47,6 +47,8 @@ class User < ApplicationRecord
   include User::OneTimePassword
   include User::Slug
   include User::Survey
+  include Rails.application.routes.url_helpers
+  include LinkToHelper
 
   DEFAULT_CONTENT_LIMITS = {
     info_requests: AlaveteliConfiguration.max_requests_per_user_per_day,
@@ -179,7 +181,9 @@ class User < ApplicationRecord
   validate :email_and_name_are_valid
 
   after_initialize :set_defaults
-  after_update :reindex_referencing_models, :update_pro_account
+  after_update :reindex_referencing_models,
+               :update_pro_account,
+               :invalidate_cached_pages
 
   acts_as_xapian texts: [:name, :about_me],
                  values: [
@@ -341,6 +345,10 @@ def expire_comments
     comments.find_each(&:reindex_request_events)
   end
 
+  def invalidate_cached_pages
+    NotifyCacheJob.perform_later(self)
+  end
+
   def locale
     (super || AlaveteliLocalization.locale).to_s
   end
@@ -673,6 +681,12 @@ def flipper_id
     "User;#{id}"
   end
 
+  def cached_urls
+    [
+      user_path(self)
+    ]
+  end
+
   private
 
   def set_defaults

config/general.yml-example

@@ -627,6 +627,20 @@ EXCEPTION_NOTIFICATIONS_TO:
 # ---
 MAX_REQUESTS_PER_USER_PER_DAY: 6
 
+# If you're running behind Varnish set this to work out where to send purge
+# requests. Otherwise, don't set it.
+#
+# VARNISH_HOSTS - Array of Strings (default: nil)
+#
+# Examples:
+#
+#   VARNISH_HOSTS:
+#     - host1
+#     - host2
+#
+# ---
+VARNISH_HOSTS: null
+
 # Adding a value here will enable Google Analytics on all non-admin pages for
 # non-admin users.
 #

lib/configuration.rb

@@ -131,6 +131,7 @@ module AlaveteliConfiguration
       USE_MAILCATCHER_IN_DEVELOPMENT: true,
       USER_SIGN_IN_ACTIVITY_RETENTION_DAYS: 0,
       UTILITY_SEARCH_PATH: ['/usr/bin', '/usr/local/bin'],
+      VARNISH_HOSTS: [],
       WORKING_OR_CALENDAR_DAYS: 'working'
     }
     # rubocop:enable Layout/LineLength

spec/controllers/admin/foi_attachments_controller_spec.rb

@@ -78,6 +78,7 @@
       end
 
       it 'should log an "edit_attachment" event on the info_request' do
+        expect(NotifyCacheJob).to receive(:perform_later).with(attachment)
         allow(@controller).to receive(:admin_current_user).
           and_return("Admin user")
 

spec/controllers/admin_incoming_message_controller_spec.rb

@@ -17,6 +17,7 @@
     end
 
     it "destroys the ActiveStorage attachment record" do
+      expect(NotifyCacheJob).to receive(:perform_later).with(@im.info_request)
       file = @im.raw_email.file
       expect(file.attached?).to eq true
       post :destroy, params: { id: @im.id }
@@ -217,6 +218,8 @@ def make_request(params=@default_params)
 
     it 'should log an "edit_incoming" event on the info_request' do
       allow(@controller).to receive(:admin_current_user).and_return("Admin user")
+      expect(NotifyCacheJob).to receive(:perform_later).
+        with(@incoming.info_request)
       make_request
       @incoming.reload
       last_event = @incoming.info_request_events.last

spec/factories/info_request_events.rb

@@ -187,6 +187,11 @@
       end
     end
 
+    factory :foi_attachment_event do
+      event_type { 'edit_attachment' }
+      params { { attachment_id: 1 } }
+    end
+
   end
 
 end