From 7bda1a8854054f26cfca11245a5f466fe310211c Mon Sep 17 00:00:00 2001 From: Matthew Somerville Date: Thu, 7 Sep 2023 19:49:53 +0100 Subject: [PATCH] Add ability to ask external caches to invalidate. Add a cached_urls method to the Comment, FoiAttachment, InfoRequest, PublicBody and User models, containing the entries that those models wish to invalidate when an event is logged via log_event or when the censor rules change. Entries are either a fixed path to be purged or a regex beginning with '^' to be banned. Adds a NotifyCacheJob that for each of the object's cached_urls, for each locale, for each host in VARNISH_HOSTS, makes BAN or PURGE HTTP requests to the host, to invalidate the relevant entries. TODO: www.whatdotheyknow.com is currently hard-coded in the job. --- app/jobs/notify_cache_job.rb | 50 ++++++++++++++++++++++++++++++++++++ app/models/censor_rule.rb | 1 + app/models/comment.rb | 7 +++++ app/models/foi_attachment.rb | 6 +++++ app/models/info_request.rb | 38 +++++++++++++++++++++++++++ app/models/public_body.rb | 9 +++++++ app/models/user.rb | 6 +++++ config/general.yml-example | 14 ++++++++++ lib/configuration.rb | 1 + 9 files changed, 132 insertions(+) create mode 100644 app/jobs/notify_cache_job.rb diff --git a/app/jobs/notify_cache_job.rb b/app/jobs/notify_cache_job.rb new file mode 100644 index 00000000000..b5a6f18366f --- /dev/null +++ b/app/jobs/notify_cache_job.rb @@ -0,0 +1,50 @@ +require 'net/http' + +class Net::HTTP::Purge < Net::HTTP::Get + METHOD = 'PURGE' +end + +class Net::HTTP::Ban < Net::HTTP::Get + METHOD = 'BAN' +end + +## +# Job to notify a cache of URLs to be purged or banned, given an object +# (that must have a cached_urls method). +# +# Examples: +# NotifyCacheJob.perform(InfoRequest.first) +# NotifyCacheJob.perform(FoiAttachment.first) +# NotifyCacheJob.perform(Comment.first) +# +class NotifyCacheJob < ApplicationJob + queue_as :default + + def perform(object) + urls = object.cached_urls + locales = [''] + AlaveteliLocalization.available_locales.map { |locale| '/' + locale } + hosts = AlaveteliConfiguration.varnish_hosts + locales.each do |locale| + hosts.each do |host| + Net::HTTP.start('www.whatdotheyknow.com', 80, host, 6081) do |http| + urls.each do |url| + if url.include? '^' + request = Net::HTTP::Ban.new(url) + else + request = Net::HTTP::Purge.new(url) + end + response = http.request(request) + result = response.code + if result == "200" + Rails.logger.debug("PURGE: Purged URL #{url} at #{host}: #{result}") + else + Rails.logger.warn( + "PURGE: Unable to purge URL #{url} at #{host}: status #{result}" + ) + end + end + end + end + end + end +end diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index a3704e9b6c2..5a40589cc4d 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -75,6 +75,7 @@ def is_global? def expire_requests if info_request InfoRequestExpireJob.perform_later(info_request) + NotifyCacheJob.perform_later(info_request) elsif user InfoRequestExpireJob.perform_later(user, :info_requests) elsif public_body diff --git a/app/models/comment.rb b/app/models/comment.rb index 07b295d2713..747a66a7df6 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -193,6 +193,13 @@ def hide(editor:) end end + def cached_urls + [ + request_path(info_request), + show_user_wall_path(url_name: user.url_name) + ] + end + private def check_body_has_content diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb index 0ed0ae02293..c2de37aeae8 100644 --- a/app/models/foi_attachment.rb +++ b/app/models/foi_attachment.rb @@ -318,6 +318,12 @@ def body_as_html(dir, opts = {}) AttachmentToHTML.to_html(self, to_html_opts) end + def cached_urls + [ + request_path(incoming_message.info_request) + ] + end + private def text_type? diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 73b3b1039e3..9814ea760a9 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -52,6 +52,7 @@ class InfoRequest < ApplicationRecord include InfoRequest::TitleValidation include Taggable include Notable + include LinkToHelper admin_columns exclude: %i[title url_title], include: %i[rejected_incoming_count] @@ -1163,6 +1164,15 @@ def log_event(type, params, options = {}) if !last_event_time || (event.created_at > last_event_time) update_column(:last_event_time, event.created_at) end + if AlaveteliConfiguration.varnish_hosts.present? + if %w[comment edit_comment hide_comment].include? type + NotifyCacheJob.perform_later(params.comment) + elsif type == 'edit_attachment' + NotifyCacheJob.perform_later(params.attachment) + else + NotifyCacheJob.perform_later(self) + end + end event end @@ -1758,6 +1768,34 @@ def latest_refusals incoming_messages.select(&:refusals?).last&.refusals || [] end + def cached_urls + feed_request = TrackThing.new( + info_request: self, + track_type: 'request_updates' + ) + feed_body = TrackThing.new( + public_body: public_body, + track_type: 'public_body_updates' + ) + feed_user = TrackThing.new( + tracked_user: user, + track_type: 'user_updates' + ) + [ + '/', + public_body_path(public_body), + request_path(self), + request_details_path(self), + '^/list*', + do_track_path(feed_request, feed='feed'), + '^/feed/list/*', + do_track_path(feed_body, feed='feed'), + do_track_path(feed_user, feed='feed'), + user_path(user), + show_user_wall_path(url_name: user.url_name) + ] + end + private def self.add_conditions_from_extra_params(params, extra_params) diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 65ef2c552c5..08198676881 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -35,6 +35,7 @@ class PublicBody < ApplicationRecord include Taggable include Notable + include LinkToHelper class ImportCSVDryRun < StandardError; end @@ -911,6 +912,14 @@ def questions PublicBodyQuestion.fetch(self) end + def cached_urls + [ + public_body_path(self), + list_public_bodies_path, + '^/body/list' + ] + end + private # If the url_name has changed, then all requested_from: queries will break diff --git a/app/models/user.rb b/app/models/user.rb index c33aae63d4b..90c1b4dcf89 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -673,6 +673,12 @@ def flipper_id "User;#{id}" end + def cached_urls + [ + user_path(self) + ] + end + private def set_defaults diff --git a/config/general.yml-example b/config/general.yml-example index f577b0c71cc..89af7a927cf 100644 --- a/config/general.yml-example +++ b/config/general.yml-example @@ -627,6 +627,20 @@ EXCEPTION_NOTIFICATIONS_TO: # --- MAX_REQUESTS_PER_USER_PER_DAY: 6 +# If you're running behind Varnish set this to work out where to send purge +# requests. Otherwise, don't set it. +# +# VARNISH_HOSTS - Array of Strings (default: nil) +# +# Examples: +# +# VARNISH_HOSTS: +# - host1 +# - host2 +# +# --- +VARNISH_HOSTS: null + # Adding a value here will enable Google Analytics on all non-admin pages for # non-admin users. # diff --git a/lib/configuration.rb b/lib/configuration.rb index ad1d2845db7..926da9169ae 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -131,6 +131,7 @@ module AlaveteliConfiguration USE_MAILCATCHER_IN_DEVELOPMENT: true, USER_SIGN_IN_ACTIVITY_RETENTION_DAYS: 0, UTILITY_SEARCH_PATH: ['/usr/bin', '/usr/local/bin'], + VARNISH_HOSTS: [], WORKING_OR_CALENDAR_DAYS: 'working' } # rubocop:enable Layout/LineLength