[7955] Cross check bulk CSV trainee uploads (#4926)

* Add tests that cross-check CSV fields - check that all of the implementation field names are documented - check that all of the implementation field names are documented - check that all of the implementation field names are present as headers in the template CSV * Fix empty template CSV for bulk trainee upload Looks like these fields were split mistakenly * Correct documentation for Disability 1-9 * Fix names of fields in docs * Fix incorrect field id in docs * Appease Rubocop * Fix spec regression
DFE-Digital · Jan 8, 2025 · 9303de2 · 9303de2
1 parent c7c1b3c
commit 9303de2
Show file tree

Hide file tree

Showing 4 changed files with 136 additions and 36 deletions.
diff --git a/app/services/bulk_update/add_trainees/import_rows.rb b/app/services/bulk_update/add_trainees/import_rows.rb
@@ -49,8 +49,7 @@ def initialize(trainee_upload)
         "Lead Partner URN" => "lead_partner_urn",
         "Employing School URN" => "employing_school_urn",
         "Trainee Start Date" => "trainee_start_date",
-        "PG Apprenticeship" => "pg_apprenticeship",
-        "Start Date" => "start_date",
+        "PG Apprenticeship Start Date" => "pg_apprenticeship_start_date",
         "Fund Code" => "fund_code",
         "Funding Method" => "funding_method",
         "Training Initiative" => "training_initiative",

diff --git a/app/views/bulk_update/add_trainees/reference_docs/fields.yaml b/app/views/bulk_update/add_trainees/reference_docs/fields.yaml
@@ -1,11 +1,11 @@
 ---
-- field_name: Provider trainee ID
+- field_name: Provider Trainee ID
   technical: provider_trainee_id
   hesa_alignment: OWNSTU
   description: Provider's own internal identifier for the student.
   format: 20 character max length
   example: '"99157234"'
-- field_name: Apply ID
+- field_name: Application ID
   technical: application_id
   hesa_alignment: APPLYAPPLICATIONID
   description: Unique Application ID, also called Application Number, used by the
@@ -22,32 +22,32 @@
     throughout their experience within the sector for which HESA collects data.
   format: A HUSID MUST be a 17 digit number
   example: '"1210007145123456"'
-- field_name: First name
+- field_name: First Names
   technical: first_names
   hesa_alignment: FNAMES
   description: This field records the forenames of the student
   format: 60 character maxlength
   example: '"Smith"'
-- field_name: Last name
+- field_name: Last Name
   technical: last_name
   hesa_alignment: SURNAME
   description: This field is the student's family name.
   format: 60 character length
   example: '"Jane"'
-- field_name: Previous last name
+- field_name: Previous Last Name
   technical: previous_last_name
   hesa_alignment: PSURNAME
   description: This is the trainee's surname immediately prior to the current surname,
     if applicable.
   format: 60 character length
   example: '"Jordan-smith"'
-- field_name: Date of birth
+- field_name: Date of Birth
   technical: date_of_birth
   hesa_alignment: BIRTHDTE
   description: This field records the date of birth of the student.
   format: YYYY-MM-DD
   example: '"1985-02-13"'
-- field_name: National Insurance number
+- field_name: NI Number
   technical: ni_number
   hesa_alignment: NIN
   description: This field describes the trainee's National Insurance Number.
@@ -57,7 +57,7 @@
     A National Insurance number must be made up of 2 letters, 6 numbers and a final
     letter, which is always A, B, C, or D.
   example: '"BX5867459C"'
-- field_name: Gender
+- field_name: Sex
   technical: sex
   hesa_alignment: SEXID
   description: This field records the gender of the student.
@@ -97,8 +97,64 @@
   format: "[HESA ethnicity reference codes](https://www.hesa.ac.uk/collection/c24053/e/ethnic)"
   example: '"103" for example is the code to use if the ethnicity is Asian - Indian
     or Indian British.'
-- field_name: Disabilities
-  technical: disability1 to disability9
+- field_name: Disability 1
+  technical: disability_1
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 2
+  technical: disability_2
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 3
+  technical: disability_3
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 4
+  technical: disability_4
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 5
+  technical: disability_5
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 6
+  technical: disability_6
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 7
+  technical: disability_7
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 8
+  technical: disability_8
+  hesa_alignment: DISABLE
+  description: This field records the type of disability that a student has, on the
+    basis of the student's own self-assessment.
+  format: "[HESA disability reference codes](https://www.hesa.ac.uk/collection/c24053/e/disable)"
+  example: '"51" for example is the code to use for learning difference such as dyslexia, dyspraxia or ADHD'
+- field_name: Disability 9
+  technical: disability_9
   hesa_alignment: DISABLE
   description: This field records the type of disability that a student has, on the
     basis of the student's own self-assessment.
@@ -115,7 +171,7 @@
 
     202 Both professional status and academic award
   example: '"201" for example is the code to use if it is Professional status only'
-- field_name: Training route
+- field_name: Training Route
   technical: training_route
   hesa_alignment: ENTRYRTE
   description: This field describes the route by which the student has accessed Initial
@@ -130,16 +186,34 @@
     as a result of successful completion of studies.
   format: "[HESA qualification aim reference codes](https://www.hesa.ac.uk/collection/c24053/e/qlaim)"
   example: '"007" for example is the code used for a BA qualification'
-- field_name: Course ITT subject
-  technical: course_subject_one, course_subject_two, course_subject_three
+- field_name: Course Subject One
+  technical: course_subject_one
+  hesa_alignment: SBJCA
+  description: |-
+    Subject of first ITT course 
+
+    Notes\: Where trainees continue to study a primary specialism, a valid HECoS code should be used in this field
+  format: "[HESA subject codes](https://www.hesa.ac.uk/collection/c24053/e/sbjca)"
+  example: '"100048" for example is the code used for design as a subject'
+- field_name: Course Subject Two
+  technical: course_subject_two
+  hesa_alignment: SBJCA
+  description: |-
+    Subject of second ITT course 
+
+    Notes\: Where trainees continue to study a primary specialism, a valid HECoS code should be used in this field
+  format: "[HESA subject codes](https://www.hesa.ac.uk/collection/c24053/e/sbjca)"
+  example: '"100048" for example is the code used for design as a subject'
+- field_name: Course Subject Three
+  technical: course_subject_three
   hesa_alignment: SBJCA
   description: |-
-    Subject of ITT course 
+    Subject of third ITT course 
 
     Notes\: Where trainees continue to study a primary specialism, a valid HECoS code should be used in this field
   format: "[HESA subject codes](https://www.hesa.ac.uk/collection/c24053/e/sbjca)"
   example: '"100048" for example is the code used for design as a subject'
-- field_name: Study mode
+- field_name: Study Mode
   technical: study_mode
   hesa_alignment: MODE
   description: This field indicates the method by which a student is being taught
@@ -155,21 +229,21 @@
 
     64 Dormant - previously part-time
   example: '"01"'
-- field_name: ITT start date
+- field_name: ITT Start Date
   technical: itt_start_date
   hesa_alignment: ITTSTARTDATE
   description: This field indicates the date the ITT course (to gain QTS) officially
     start
   format: YYYY-MM-DD
   example: '"2024-09-01"'
-- field_name: ITT end date
+- field_name: ITT End Date
   technical: itt_end_date
   hesa_alignment: EXPECTEDENDDATE
   description: This field records the expected end date for the engagement
   format: YYYY-MM-DD
   example: YYYY-MM-DD
-- field_name: Course age range
-  technical: course_age_range 
+- field_name: Course Age Range
+  technical: course_age_range
   hesa_alignment: ITTPHSC
   description: This field describes in more detail the student's initial teacher
     training aim
@@ -193,7 +267,7 @@
 
     64 Dormant - previously part-time
   example: '"01"'
-- field_name: Course year
+- field_name: Course Year
   technical: course_year
   hesa_alignment: YEARPRG
   description: This field indicates the year number of the course that the student
@@ -208,16 +282,16 @@
 
     3 Third year
   example: '"1" as an example of using the code for First year.'
-- field_name: Lead partner URN
-  technical: lead_school_urn
+- field_name: Lead Partner URN
+  technical: lead_partner_urn
   hesa_alignment: SDLEAD
   description: This field collects the lead school for School Direct trainees
   format: Must be a 6-digit number
   example: |-
     "115795" 
 
     Use code 900020 if the lead partner does not have a URN
-- field_name: Employing school URN
+- field_name: Employing School URN
   technical: employing_school_urn
   hesa_alignment: SDEMPLOY
   description: This field collects the employing school for School Direct salaried
@@ -227,14 +301,14 @@
     "115795" 
 
     Use code 900020 if the school does not have a URN
-- field_name: Trainee start date
+- field_name: Trainee Start Date
   technical: trainee_start_date
   hesa_alignment: TRAINEESTARTDATE
   description: This field indicates the date the student commenced their ITT course
     where this is different to the start date of the instance as recorded in Student.COMDATE.
   format: YYYY-MM-DD
   example: "2024-09-01"
-- field_name: PG apprenticeship start date
+- field_name: PG Apprenticeship Start Date
   technical: pg_apprenticeship_start_date
   hesa_alignment: PGAPPSTDT
   description: This records the start date of a postgraduate's teaching apprenticeship.
@@ -259,7 +333,7 @@
 
     7 Eligible for funding from the DfE
   example: '"2"'
-- field_name: Funding method
+- field_name: Funding Method
   technical: funding_method
   hesa_alignment: BURSLEV
   description: This field records the training bursary awarded to the student. Bursary
@@ -279,7 +353,7 @@
 
     E Grant
   example: '"E"'
-- field_name: Training initiative
+- field_name: Training Initiative
   technical: training_initiative
   hesa_alignment: INITIATIVES
   description: This field identifies students who are part of a specific scheme that
@@ -309,7 +383,7 @@
     011 Primary with mathematics specialist
 
     019 Additional ITT place for PE with a priority subject
-- field_name: Additional training initiative
+- field_name: Additional Training Initiative
   technical: additional_training_initiative
   hesa_alignment: INITIATIVES
   description: This field records the student identifier which is to be unique to

diff --git a/spec/fixtures/files/bulk_update/trainee_uploads/five_trainees.csv b/spec/fixtures/files/bulk_update/trainee_uploads/five_trainees.csv
@@ -1,6 +1,6 @@
-Provider Trainee ID,Application ID,HESA ID,First Names,Last Name,Previous Last Name,Date of Birth,NI Number,Sex,Email,Nationality,Ethnicity,Disability 1,Disability 2,Disability 3,Disability 4,Disability 5,Disability 6,Disability 7,Disability 8,Disability 9,ITT Aim,Training Route,Qualification Aim,Course Subject One,Course Subject Two,Course Subject Three,Study Mode,ITT Start Date,ITT End Date,Course Age Range,Course Year,Lead Partner URN,Employing School URN,Trainee Start Date,PG Apprenticeship,Start Date,Fund Code,Funding Method,Training Initiative,Additional Training Initiative
-1198477681,,60114404165089097,Jonas,Padberg,Koelpin,1964-03-07,,10,[email protected],GB,,,,,,,,,,,202,11,3,100511,,,63,2023-01-01,2023-10-01,13917,2023,,,,,,7,4,,
-6690742641,,71489126120547141,Myriam,Bruen,Stokes,1993-04-01,,10,[email protected],GB,,,,,,,,,,,201,11,8,100511,,,63,2023-01-01,2023-10-01,13918,2023,,,,,,7,4,,
-8920767048,,45471004454825701,Usha,Rolfson,Nikolaus,1989-08-12,,99,[email protected],GB,,,,,,,,,,,201,11,8,100511,,,63,2023-01-01,2023-10-01,13911,2023,,,,,,2,4,,
-7840305304,,57760321187383067,Fidel,Hessel,Mayert,1980-08-01,,11,[email protected],GB,,,,,,,,,,,201,11,28,100511,,,63,2023-01-01,2023-10-01,13915,2023,,,,,,2,4,,
-5112140029,,92670443173284377,Melony,Kilback,Schmidt,2001-08-29,,12,[email protected],GB,,,,,,,,,,,201,11,32,100511,,,63,2023-01-01,2023-10-01,13909,2023,,,,,,7,4,,
+Provider Trainee ID,Application ID,HESA ID,First Names,Last Name,Previous Last Name,Date of Birth,NI Number,Sex,Email,Nationality,Ethnicity,Disability 1,Disability 2,Disability 3,Disability 4,Disability 5,Disability 6,Disability 7,Disability 8,Disability 9,ITT Aim,Training Route,Qualification Aim,Course Subject One,Course Subject Two,Course Subject Three,Study Mode,ITT Start Date,ITT End Date,Course Age Range,Course Year,Lead Partner URN,Employing School URN,Trainee Start Date,PG Apprenticeship Start Date,Fund Code,Funding Method,Training Initiative,Additional Training Initiative
+1198477681,,60114404165089097,Jonas,Padberg,Koelpin,1964-03-07,,10,[email protected],GB,,,,,,,,,,,202,11,3,100511,,,63,2023-01-01,2023-10-01,13917,2023,,,,,7,4,,
+6690742641,,71489126120547141,Myriam,Bruen,Stokes,1993-04-01,,10,[email protected],GB,,,,,,,,,,,201,11,8,100511,,,63,2023-01-01,2023-10-01,13918,2023,,,,,7,4,,
+8920767048,,45471004454825701,Usha,Rolfson,Nikolaus,1989-08-12,,99,[email protected],GB,,,,,,,,,,,201,11,8,100511,,,63,2023-01-01,2023-10-01,13911,2023,,,,,2,4,,
+7840305304,,57760321187383067,Fidel,Hessel,Mayert,1980-08-01,,11,[email protected],GB,,,,,,,,,,,201,11,28,100511,,,63,2023-01-01,2023-10-01,13915,2023,,,,,2,4,,
+5112140029,,92670443173284377,Melony,Kilback,Schmidt,2001-08-29,,12,[email protected],GB,,,,,,,,,,,201,11,32,100511,,,63,2023-01-01,2023-10-01,13909,2023,,,,,7,4,,
diff --git a/spec/services/bulk_update/add_trainees/import_rows_spec.rb b/spec/services/bulk_update/add_trainees/import_rows_spec.rb
@@ -130,6 +130,33 @@ module AddTrainees
           end
         end
       end
+
+      describe "documentation cross-checks" do
+        let(:documentation_fields) { YAML.load_file(CsvFields::View::FIELD_DEFINITION_PATH) }
+
+        BulkUpdate::AddTrainees::ImportRows::TRAINEE_HEADERS.each_value do |id|
+          it "documents field with id #{id}" do
+            expect(documentation_fields.map { |field| field["technical"] }).to include(id)
+          end
+        end
+
+        BulkUpdate::AddTrainees::ImportRows::TRAINEE_HEADERS.each_key do |name|
+          it "documents field with name #{name}" do
+            expect(documentation_fields.map { |field| field["field_name"] }).to include(name)
+          end
+        end
+      end
+
+      describe "template CSV cross-checks" do
+        let(:file_path) { Rails.public_path.join("csv/bulk_create_trainee.csv") }
+        let(:headers) { CSVSafe.new(File.open(file_path), headers: true, encoding: "UTF-8").read.headers }
+
+        BulkUpdate::AddTrainees::ImportRows::TRAINEE_HEADERS.each_key do |name|
+          it "includes column header #{name}" do
+            expect(headers).to include(name)
+          end
+        end
+      end
     end
   end
 end