[7957] Update HESA placements import (#4999)

* Update HESA placements import
- support NOT_APPLICABLE_SCHOOL_URNS
- provide feedback from rake task
* Update tests

app/services/placements/import_from_csv.rb

@@ -4,27 +4,45 @@ module Placements
   class ImportFromCsv
     include ServicePattern
 
+    attr_reader :upload, :unmatched_hesa_ids, :unmatched_urns
+
     def initialize(upload_id:)
       @upload = Upload.find(upload_id)
+      @unmatched_hesa_ids = []
+      @unmatched_urns = []
     end
 
-    # Uploaded CSV file should contain hesa_id and urn headers only
     def call
-      table = CSV.parse(upload.file.download, headers: true)
-      table.each do |row|
-        next unless (school = School.find_by(urn: row["urn"]))
-        next unless (trainee = matching_trainee_in_previous_cycle(row["hesa_id"]))
+      # Uploaded CSV file should contain hesa_id and urn headers only
+      placement_data = CSV.parse(upload.file.download, headers: true, header_converters: :symbol)
 
-        Placement.find_or_create_by(trainee:, school:)
+      placement_data.each do |row|
+        if (trainee = matching_trainee_in_previous_cycle(row[:hesa_id]))
+          urn = row[:urn]
+          if valid_unknown_school_urn?(urn)
+            Placement.find_or_create_by!(trainee: trainee, urn: urn, name: I18n.t("components.placement_detail.magic_urn.#{urn}"))
+          elsif (school = School.find_by(urn:))
+            Placement.find_or_create_by(trainee:, school:)
+          else
+            unmatched_urns << urn
+          end
+        else
+          unmatched_hesa_ids << row[:hesa_id]
+        end
       end
+
+      self
     end
 
   private
 
-    attr_reader :upload
-
     def matching_trainee_in_previous_cycle(hesa_id)
       AcademicCycle.previous.total_trainees.find_by(hesa_id:)
     end
+
+    def valid_unknown_school_urn?(urn)
+      # Check if the urn is one of the HESA codes for not applicable school URNs
+      urn.in?(Trainees::CreateFromHesa::NOT_APPLICABLE_SCHOOL_URNS)
+    end
   end
 end

lib/tasks/import_hesa_placements.rake

@@ -7,6 +7,21 @@ namespace :import_hesa_placements do
   desc "Import HESA placements from uploaded CSV"
   task :from_upload, [:upload_id] => :environment do |_, args|
     upload_id = args.upload_id.to_i
-    Placements::ImportFromCsv.call(upload_id:)
+    import = Placements::ImportFromCsv.call(upload_id:)
+
+    puts("")
+    puts("HESA ids which don't match any trainees in Register:")
+    import.unmatched_hesa_ids.uniq.each do |hesa_id|
+      puts("  hesa_id: #{hesa_id}")
+    end
+
+    puts("")
+    puts("URNs which don't match any Schools in Register:")
+    import.unmatched_urns.uniq.each do |urn|
+      puts("  URN: #{urn}")
+    end
+
+    puts("")
+    puts("Total number of rows not imported: #{(import.unmatched_hesa_ids.count + import.unmatched_urns.count).to_fs(:delimited)}")
   end
 end

spec/fixtures/files/placements_import.csv

@@ -2,5 +2,6 @@ hesa_id,urn
 0000083799785,137335
 1810078035519,145787
 1810078035519,137335
-2010070003610,143956
+1810078035519,143956
 2820078035529,137335
+22100027187725416,900000

spec/services/placements/import_from_csv_spec.rb

@@ -8,6 +8,8 @@ module Placements
       let(:upload) { create(:upload, fixture_name: "placements_import.csv") }
       let!(:school1) { create(:school, urn: "137335") }
       let!(:school2) { create(:school, urn: "145787") }
+      let!(:valid_unknown_school_urn) { "900000" }
+      let!(:name) { I18n.t("components.placement_detail.magic_urn.#{valid_unknown_school_urn}") }
       let!(:trainee1) do
         create(
           :trainee,
@@ -32,29 +34,48 @@ module Placements
           itt_end_date: this_cycle.end_date,
         )
       end
+      let!(:trainee4) do
+        create(
+          :trainee,
+          hesa_id: "22100027187725416",
+          itt_start_date: last_cycle.start_date,
+          itt_end_date: last_cycle.end_date,
+        )
+      end
       let(:last_cycle) { create(:academic_cycle, previous_cycle: true) }
       let(:this_cycle) { create(:academic_cycle, :current) }
 
       it "creates placements for each row in the csv where there are matching school and trainee" do
         expect { described_class.call(upload_id: upload.id) }
-          .to change(Placement, :count).by(3)
+          .to change(Placement, :count).by(4)
       end
 
       it "creates placements with the correct attributes" do
         described_class.call(upload_id: upload.id)
 
-        expect(Placement.pluck(:school_id)).to contain_exactly(
+        expect(Placement.pluck(:school_id).compact).to contain_exactly(
           school1.id, school2.id, school1.id
         )
         expect(Placement.pluck(:trainee_id)).to contain_exactly(
-          trainee1.id, trainee2.id, trainee2.id
+          trainee1.id, trainee2.id, trainee2.id, trainee4.id
+        )
+        expect(Placement.pluck(:urn).compact).to contain_exactly(
+          valid_unknown_school_urn,
+        )
+        expect(Placement.pluck(:name).compact).to contain_exactly(
+          name,
         )
       end
 
-      it "does not create placements where there is no matching school" do
+      it "does not create placements where there is no matching school (unless a valid HESA code)" do
         described_class.call(upload_id: upload.id)
 
-        expect(Placement.all.map(&:school).map(&:urn)).not_to include("143956")
+        expect(Placement.where.not(school: nil).map(&:school).map(&:urn)).not_to include("143956")
+      end
+
+      it "adds unmatched urns to the unmatched_urns array" do
+        import = described_class.call(upload_id: upload.id)
+        expect(import.unmatched_urns).to include("143956")
       end
 
       it "only creates placements for trainees in the last cycle" do
@@ -69,11 +90,18 @@ module Placements
         expect(Placement.all.map(&:trainee).map(&:hesa_id)).not_to include("2010070003610")
       end
 
+      it "creates placements where the urn is one of the HESA codes for not applicable school URNs" do
+        described_class.call(upload_id: upload.id)
+
+        expect(Placement.where(urn: valid_unknown_school_urn).first.trainee_id).to eq(trainee4.id)
+      end
+
       context "matching placement already exists" do
         before do
           create(:placement, school: school1, trainee: trainee1)
           create(:placement, school: school2, trainee: trainee2)
           create(:placement, school: school1, trainee: trainee2)
+          create(:placement, urn: valid_unknown_school_urn, name: name, trainee: trainee4)
         end
 
         it "does not create duplicate placements" do