From d4923b4583b474169a967bd872b69f1b478a453b Mon Sep 17 00:00:00 2001 From: Matthew Somerville Date: Thu, 7 Sep 2023 19:49:53 +0100 Subject: [PATCH] wip cache banning No expectation this works like this, is basically pseudo-code in parts. --- app/jobs/notify_cache_job.rb | 47 ++++++++++++++++++++++++++++++++++++ app/models/censor_rule.rb | 1 + app/models/comment.rb | 7 ++++++ app/models/foi_attachment.rb | 6 +++++ app/models/info_request.rb | 30 +++++++++++++++++++++++ app/models/public_body.rb | 8 ++++++ app/models/user.rb | 6 +++++ config/general.yml-example | 14 +++++++++++ lib/configuration.rb | 1 + 9 files changed, 120 insertions(+) create mode 100644 app/jobs/notify_cache_job.rb diff --git a/app/jobs/notify_cache_job.rb b/app/jobs/notify_cache_job.rb new file mode 100644 index 00000000000..0ee9fc4686c --- /dev/null +++ b/app/jobs/notify_cache_job.rb @@ -0,0 +1,47 @@ +require 'net/http' + +class Net::HTTP::Purge < Net::HTTP::Get + METHOD = 'PURGE' +end + +class Net::HTTP::Ban < Net::HTTP::Get + METHOD = 'BAN' +end + +## +# Job to notify a cache of URLs to be purged or banned, given an object +# (that must have a cached_urls method). +# +# Examples: +# NotifyCacheJob.perform(InfoRequest.first) +# NotifyCacheJob.perform(FoiAttachment.first) +# NotifyCacheJob.perform(Comment.first) +# +class NotifyCacheJob < ApplicationJob + queue_as :default + + def perform(object) + urls = object.cached_urls + hosts = AlaveteliConfiguration.varnish_hosts + hosts.each do |host| + Net::HTTP.start('www.whatdotheyknow.com', 80, host, 6081) do |http| + urls.each do |url| + if url.include? '^' + request = Net::HTTP::Ban.new(url) + else + request = Net::HTTP::Purge.new(url) + end + response = http.request(request) + result = response.code + if result == "200" + Rails.logger.debug("PURGE: Purged URL #{url} at #{host}: #{result}") + else + Rails.logger.warn( + "PURGE: Unable to purge URL #{url} at #{host}: status #{result}" + ) + end + end + end + end + end +end diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index a3704e9b6c2..5a40589cc4d 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -75,6 +75,7 @@ def is_global? def expire_requests if info_request InfoRequestExpireJob.perform_later(info_request) + NotifyCacheJob.perform_later(info_request) elsif user InfoRequestExpireJob.perform_later(user, :info_requests) elsif public_body diff --git a/app/models/comment.rb b/app/models/comment.rb index 07b295d2713..f470c906aa5 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -208,4 +208,11 @@ def check_body_uses_mixed_capitals errors.add(:body, msg) end end + + def cached_urls + [ + request_path(info_request), + show_user_wall_path(url_name: user.url_name) + ] + end end diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb index 0ed0ae02293..0e7a4233b39 100644 --- a/app/models/foi_attachment.rb +++ b/app/models/foi_attachment.rb @@ -323,4 +323,10 @@ def body_as_html(dir, opts = {}) def text_type? AlaveteliTextMasker::TextMask.include?(content_type) end + + def cached_urls + [ + request_path(incoming_message.info_request) + ] + end end diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 73b3b1039e3..da9e679a910 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -1163,6 +1163,15 @@ def log_event(type, params, options = {}) if !last_event_time || (event.created_at > last_event_time) update_column(:last_event_time, event.created_at) end + if AlaveteliConfiguration.varnish_hosts.present? + if %w[comment edit_comment hide_comment].include? type + NotifyCacheJob.perform_later(params.comment) + elsif type == 'edit_attachment' + NotifyCacheJob.perform_later(params.attachment) + else + NotifyCacheJob.perform_later(self) + end + end event end @@ -1904,4 +1913,25 @@ def reindexable_attribute_changed? saved_change_to_attribute?(attr) end end + + def cached_urls + [ + '/', + public_body_path(body), + request_path(self), + request_details_path(self), + '^/list*', + do_track_path({ track_type => 'request_updates', info_request => self }, + feed='feed'), + '^/feed/list/*', + do_track_path({ track_type => 'public_body_updates', public_body => public_body }, + feed='feed'), + do_track_path({ track_type => 'user_updates', tracked_user => user }, + feed='feed'), + user_path(user), + show_user_wall_path(url_name: user.url_name), + public_body_path(self), + '^/body/list' + ] + end end diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 65ef2c552c5..58f890a060d 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -968,4 +968,12 @@ def update_missing_email_tag def missing_email? !has_request_email? end + + def cached_urls + [ + public_body_path(self), + list_public_bodies_path, + '^/body/list' + ] + end end diff --git a/app/models/user.rb b/app/models/user.rb index c33aae63d4b..16e4d6de5bb 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -713,4 +713,10 @@ def update_pro_account def content_limit(content) content_limits[content] end + + def cached_urls + [ + user_path(self) + ] + end end diff --git a/config/general.yml-example b/config/general.yml-example index f577b0c71cc..89af7a927cf 100644 --- a/config/general.yml-example +++ b/config/general.yml-example @@ -627,6 +627,20 @@ EXCEPTION_NOTIFICATIONS_TO: # --- MAX_REQUESTS_PER_USER_PER_DAY: 6 +# If you're running behind Varnish set this to work out where to send purge +# requests. Otherwise, don't set it. +# +# VARNISH_HOSTS - Array of Strings (default: nil) +# +# Examples: +# +# VARNISH_HOSTS: +# - host1 +# - host2 +# +# --- +VARNISH_HOSTS: null + # Adding a value here will enable Google Analytics on all non-admin pages for # non-admin users. # diff --git a/lib/configuration.rb b/lib/configuration.rb index ad1d2845db7..e4251b026f0 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -131,6 +131,7 @@ module AlaveteliConfiguration USE_MAILCATCHER_IN_DEVELOPMENT: true, USER_SIGN_IN_ACTIVITY_RETENTION_DAYS: 0, UTILITY_SEARCH_PATH: ['/usr/bin', '/usr/local/bin'], + VARNISH_HOSTS: nil, WORKING_OR_CALENDAR_DAYS: 'working' } # rubocop:enable Layout/LineLength