diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 8be5fd9a05..3141f5ab21 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -32,6 +32,7 @@ class RouteNotFound < StandardError include FastGettext::Translation # make functions like _, n_, N_ etc available) include AlaveteliPro::PostRedirectHandler + include RobotsHeaders # NOTE: a filter stops the chain if it redirects or renders something before_action :html_response diff --git a/app/controllers/attachment_masks_controller.rb b/app/controllers/attachment_masks_controller.rb index c397d8da99..6a42501449 100644 --- a/app/controllers/attachment_masks_controller.rb +++ b/app/controllers/attachment_masks_controller.rb @@ -32,10 +32,6 @@ def done private - def set_no_crawl_headers - headers['X-Robots-Tag'] = 'noindex' - end - def decode_referer @referer = verifier.verified(params[:referer]) end diff --git a/app/controllers/citations_controller.rb b/app/controllers/citations_controller.rb index 5f26fe1ac2..ae58d26c19 100644 --- a/app/controllers/citations_controller.rb +++ b/app/controllers/citations_controller.rb @@ -5,6 +5,7 @@ class CitationsController < ApplicationController before_action :authenticate before_action :load_info_request_and_authorise before_action :set_in_pro_area + before_action :set_no_crawl_headers def new @citation = current_user.citations.build diff --git a/app/controllers/concerns/prominence_headers.rb b/app/controllers/concerns/prominence_headers.rb index 22f3f550ce..befc48039f 100644 --- a/app/controllers/concerns/prominence_headers.rb +++ b/app/controllers/concerns/prominence_headers.rb @@ -24,7 +24,7 @@ def set_normal_headers end def set_backpage_headers - headers['X-Robots-Tag'] = 'noindex' + set_no_crawl_headers end def set_requester_only_headers diff --git a/app/controllers/concerns/public_tokenable.rb b/app/controllers/concerns/public_tokenable.rb index 32f2795bae..8a80823e44 100644 --- a/app/controllers/concerns/public_tokenable.rb +++ b/app/controllers/concerns/public_tokenable.rb @@ -8,16 +8,12 @@ module PublicTokenable extend ActiveSupport::Concern included do - before_action :set_no_crawl_headers + before_action :set_no_crawl_headers, if: -> { public_token } helper_method :public_token end private - def set_no_crawl_headers - headers['X-Robots-Tag'] = 'noindex' if public_token - end - def public_token params[:public_token] end diff --git a/app/controllers/concerns/robots_headers.rb b/app/controllers/concerns/robots_headers.rb new file mode 100644 index 0000000000..79f6d53232 --- /dev/null +++ b/app/controllers/concerns/robots_headers.rb @@ -0,0 +1,17 @@ +## +# Set robots noindex headers. +# +module RobotsHeaders + extend ActiveSupport::Concern + + included do + before_action :set_no_crawl_headers, if: -> { params[:page].to_i > 1 } + end + + private + + def set_no_crawl_headers + @no_crawl = true + headers['X-Robots-Tag'] = 'noindex, nofollow' + end +end diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb index 2214f4ccff..3df37507ad 100644 --- a/app/controllers/request_controller.rb +++ b/app/controllers/request_controller.rb @@ -15,6 +15,7 @@ class RequestController < ApplicationController before_action :redirect_new_form_to_pro_version, only: [:select_authority, :new] before_action :set_in_pro_area, only: [:select_authority, :show] before_action :setup_results_pagination, only: [:list, :similar] + before_action :set_no_crawl_headers, only: [:new, :details, :similar] helper_method :state_transitions_empty? @@ -470,9 +471,6 @@ def setup_results_pagination @per_page = PER_PAGE @max_results = MAX_RESULTS - # Don't let robots go more than 20 pages in - @no_crawl = true if @page > 20 - # Later pages are very expensive to load return if @page <= MAX_RESULTS / PER_PAGE diff --git a/app/controllers/user_controller.rb b/app/controllers/user_controller.rb index 3dd0977586..4f048cd78b 100644 --- a/app/controllers/user_controller.rb +++ b/app/controllers/user_controller.rb @@ -18,6 +18,7 @@ class UserController < ApplicationController before_action :work_out_post_redirect, only: [ :signup ] before_action :set_request_from_foreign_country, only: [ :signup ] before_action :set_in_pro_area, only: [ :signup ] + before_action :set_no_crawl_headers, only: :wall # Normally we wouldn't be verifying the authenticity token on these actions # anyway as there shouldn't be a user_id in the session when the before diff --git a/app/views/request/_after_actions.html.erb b/app/views/request/_after_actions.html.erb index 0b187ecd7c..784fa66a0e 100644 --- a/app/views/request/_after_actions.html.erb +++ b/app/views/request/_after_actions.html.erb @@ -14,20 +14,20 @@ @@ -42,7 +42,7 @@ @@ -61,7 +61,7 @@ <% else %>
  • - <%= link_to _("Report this request"), new_request_report_path(info_request.url_title) %> + <%= link_to _("Report this request"), new_request_report_path(info_request.url_title), rel: 'nofollow' %> <%= link_to _("Help"), help_about_path(:anchor => "reporting") %>
    • @@ -69,7 +69,7 @@ <% if feature_enabled?(:annotations) && info_request.comments_allowed? %>
  • - <%= link_to _('Add an annotation'), new_comment_path(:url_title => info_request.url_title) %> + <%= link_to _('Add an annotation'), new_comment_path(:url_title => info_request.url_title), rel: 'nofollow' %>
    • <% end %> @@ -80,11 +80,11 @@ <% end %>
  • - <%= link_to _("Download a zip file of all correspondence"), download_entire_request_path(:url_title => info_request.url_title) %> + <%= link_to _("Download a zip file of all correspondence"), download_entire_request_path(:url_title => info_request.url_title), rel: 'nofollow' %>
  • - <%= link_to _('View event history details'), request_details_path(info_request) %> + <%= link_to _('View event history details'), request_details_path(info_request), rel: 'nofollow' %>
  • diff --git a/doc/CHANGES.md b/doc/CHANGES.md index b48aa4fcb8..ec3c7c3a9f 100644 --- a/doc/CHANGES.md +++ b/doc/CHANGES.md @@ -2,6 +2,8 @@ ## Highlighted Features +* Update actions and pages which set "noindex", "nofollow" crawler directives + (Graeme Porteous) * Add default value and not null constraint to `CensorRule#regexp` (Gareth Rees) * Allow requests to be listed and filtered by tag (Graeme Porteous) * Fix admin error when authority are missing an email address (Graeme Porteous) diff --git a/spec/controllers/attachment_masks_controller_spec.rb b/spec/controllers/attachment_masks_controller_spec.rb index 5d6b759671..8eb46254c6 100644 --- a/spec/controllers/attachment_masks_controller_spec.rb +++ b/spec/controllers/attachment_masks_controller_spec.rb @@ -54,9 +54,9 @@ def wait expect(response).to render_template(:wait) end - it 'sets noindex header' do + it 'sets noindex, nofollow header' do wait - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end end @@ -118,9 +118,9 @@ def done expect(response).to render_template(:done) end - it 'sets noindex header' do + it 'sets noindex, nofollow header' do done - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end end diff --git a/spec/controllers/attachments_controller_spec.rb b/spec/controllers/attachments_controller_spec.rb index cf2b870806..29574ea13c 100644 --- a/spec/controllers/attachments_controller_spec.rb +++ b/spec/controllers/attachments_controller_spec.rb @@ -198,13 +198,13 @@ def show(params = {}) expect(assigns(:info_request)).to eq(info_request) end - it 'adds noindex header when using public token' do + it 'adds noindex, nofollow header when using public token' do expect(InfoRequest).to receive(:find_by!).with(public_token: 'ABC'). and_return(info_request) show(public_token: 'ABC', id: nil) - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end it 'passes public token to current ability' do @@ -287,14 +287,14 @@ def show(params = {}) context 'when the request is backpage' do let(:request_prominence) { 'backpage' } - it 'sets a noindex header when viewing' do + it 'sets a noindex, nofollow header when viewing' do show - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end - it 'sets a noindex header when viewing a cached copy' do + it 'sets a noindex, nofollow header when viewing a cached copy' do show - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end context 'when logged in as requester' do @@ -449,13 +449,13 @@ def show_as_html(params = {}) expect(assigns(:info_request)).to eq(info_request) end - it 'adds noindex header when using public token' do + it 'adds noindex, nofollow header when using public token' do expect(InfoRequest).to receive(:find_by!).with(public_token: '123'). and_return(info_request) show_as_html(public_token: '123', id: nil) - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end context 'when attachment has not been masked' do @@ -560,14 +560,14 @@ def show_as_html(params = {}) context 'when the request is backpage' do let(:request_prominence) { 'backpage' } - it 'sets a noindex header when viewing a HTML version' do + it 'sets a noindex, nofollow header when viewing a HTML version' do show_as_html - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end - it 'sets a noindex header when viewing a cached HTML version' do + it 'sets a noindex, nofollow header when viewing a cached HTML version' do show_as_html - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end end diff --git a/spec/controllers/citations_controller_spec.rb b/spec/controllers/citations_controller_spec.rb index bc39876f76..4d48ad5d30 100644 --- a/spec/controllers/citations_controller_spec.rb +++ b/spec/controllers/citations_controller_spec.rb @@ -37,6 +37,11 @@ expect(response).to be_successful end end + + it 'adds noindex, nofollow header' do + action + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' + end end # when requester diff --git a/spec/controllers/public_tokens_controller_spec.rb b/spec/controllers/public_tokens_controller_spec.rb index cf3b646189..77ab6927c4 100644 --- a/spec/controllers/public_tokens_controller_spec.rb +++ b/spec/controllers/public_tokens_controller_spec.rb @@ -25,9 +25,9 @@ expect(assigns(:info_request)).to eq info_request end - it 'adds noindex header' do + it 'adds noindex, nofollow header' do get :show, params: { public_token: 'TOKEN' } - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end it 'returns http success' do diff --git a/spec/controllers/request_controller_spec.rb b/spec/controllers/request_controller_spec.rb index aea2f87221..5b7b2b7ee9 100644 --- a/spec/controllers/request_controller_spec.rb +++ b/spec/controllers/request_controller_spec.rb @@ -603,9 +603,9 @@ def expect_hidden(hidden_template) expect(response).to render_template('show') end - it 'sets a noindex header' do + it 'sets a noindex, nofollow header' do get :show, params: { url_title: info_request.url_title } - expect(response.headers['X-Robots-Tag']).to eq 'noindex' + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' end end end @@ -734,6 +734,11 @@ def expect_hidden(hidden_template) expect(response).to render_template('new_bad_contact') end + it 'adds noindex, nofollow header' do + get :new, params: { public_body_id: @body.id } + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' + end + context "the outgoing message includes an email address" do context "there is no logged in user" do it "displays a flash error message without escaping the HTML" do @@ -1964,6 +1969,11 @@ def send_request get :similar, params: { url_title: badger_request.url_title } }.to raise_error(ActiveRecord::RecordNotFound) end + + it 'adds noindex, nofollow header' do + get :similar, params: { url_title: badger_request.url_title } + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' + end end RSpec.describe RequestController, "when the site is in read_only mode" do @@ -2041,6 +2051,11 @@ def send_request }.to raise_error(ActiveRecord::RecordNotFound) end end + + it 'adds noindex, nofollow header' do + get :details, params: { url_title: info_request.url_title } + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' + end end end diff --git a/spec/controllers/user_controller_spec.rb b/spec/controllers/user_controller_spec.rb index 654dfeff75..4747ee9d6e 100644 --- a/spec/controllers/user_controller_spec.rb +++ b/spec/controllers/user_controller_spec.rb @@ -1351,4 +1351,10 @@ def make_request get :wall, params: { url_name: user.url_name } expect(assigns[:feed_results]).to be_empty end + + it 'adds noindex, nofollow header' do + user = FactoryBot.create(:user) + get :wall, params: { url_name: user.url_name } + expect(response.headers['X-Robots-Tag']).to eq 'noindex, nofollow' + end end