Merge branch 'post-attachment-background-processing' into develop

.gitignore

@@ -18,6 +18,7 @@
 /config/memcached.yml
 /config/newrelic.yml
 /config/rails_env.rb
+/config/sidekiq.yml
 /config/storage.yml
 /config/user_spam_scorer.yml
 /config/xapian.yml

app/jobs/foi_attachment_mask_job.rb

@@ -31,6 +31,10 @@ def perform(attachment)
     end
 
     attachment.update(body: body, masked_at: Time.zone.now)
+
+  rescue MailHandler::MismatchedAttachmentHexdigest
+    incoming_message.parse_raw_email!(true)
+    retry_job
   end
 
   private

config/general.yml-example

@@ -807,6 +807,7 @@ SHARED_FILES_PATH: ''
 SHARED_FILES:
   - config/database.yml
   - config/general.yml
+  - config/sidekiq.yml
   - config/storage.yml
   - config/rails_env.rb
   - config/httpd.conf

docker/bootstrap

@@ -30,6 +30,7 @@ success_msg 'done'
 
 notice_msg "Copying example files..."
 [ ! -f config/database.yml ] && cp config/database.yml-example config/database.yml
+[ ! -f config/sidekiq.yml ] && cp config/sidekiq.yml-example config/sidekiq.yml
 [ ! -f config/storage.yml ] && cp config/storage.yml-example config/storage.yml
 [ ! -f config/general-alavetelitheme.yml ] && cp config/general.yml-example config/general-alavetelitheme.yml
 [ ! -L config/general.yml ] && [ ! -f config/general.yml ] && ln -s config/general-alavetelitheme.yml config/general.yml

lib/mail_handler/backends/mail_backend.rb

@@ -40,6 +40,8 @@ module Backends
     module MailBackend
       include ConfigHelper
 
+      MismatchedAttachmentHexdigest = Class.new(StandardError)
+
       def backend
         'Mail'
       end
@@ -384,7 +386,11 @@ def attachment_body_for_hexdigest(mail, hexdigest:)
         attributes = get_attachment_attributes(mail).find do |attrs|
           attrs[:hexdigest] == hexdigest
         end
-        attributes&.fetch(:body)
+
+        return attributes.fetch(:body) if attributes
+
+        raise MismatchedAttachmentHexdigest,
+          "can't find attachment matching hexdigest: #{hexdigest}"
       end
 
       # Format

script/install-as-user

@@ -184,6 +184,9 @@ do
     fi
 done
 
+# add sidekiq.yml
+cp config/sidekiq.yml-example config/sidekiq.yml
+
 # add storage.yml
 cp config/storage.yml-example config/storage.yml
 

spec/jobs/foi_attachment_mask_job_spec.rb

@@ -5,8 +5,10 @@
   let(:incoming_message) { info_request.incoming_messages.first }
   let(:attachment) { incoming_message.foi_attachments.last }
 
+  let(:instance) { described_class.new }
+
   def perform
-    described_class.new.perform(attachment)
+    instance.perform(attachment)
   end
 
   before { rebuild_raw_emails(info_request) }
@@ -90,4 +92,22 @@ def perform
     expect(attachment.body).to_not include 'dull'
     expect(attachment.body).to include 'Horse'
   end
+
+  context 'when unmasked body is not avaliable' do
+    before do
+      allow(attachment).to receive(:unmasked_body).and_raise(
+        MailHandler::MismatchedAttachmentHexdigest
+      )
+    end
+
+    it 'reparses raw email' do
+      expect(incoming_message).to receive(:parse_raw_email!).with(true)
+      perform
+    end
+
+    it 'retries job' do
+      expect(instance).to receive(:retry_job)
+      perform
+    end
+  end
 end

spec/lib/mail_handler/backends/mail_backend_spec.rb

@@ -425,9 +425,10 @@
     end
 
     context 'non-matching hexdigest' do
-      it 'returns nil' do
-        body = attachment_body_for_hexdigest(mail, hexdigest: 'incorrect')
-        expect(body).to be_nil
+      it 'raises MismatchedAttachmentHexdigest error' do
+        expect {
+          attachment_body_for_hexdigest(mail, hexdigest: 'incorrect')
+        }.to raise_error(MailHandler::MismatchedAttachmentHexdigest)
       end
     end
   end