Skip to content

Commit

Permalink
Merge branch 'development' into library_update_2024
Browse files Browse the repository at this point in the history
  • Loading branch information
Chris-bennettWk authored Sep 13, 2024
2 parents b68c10c + 0a57f36 commit 11d0da5
Show file tree
Hide file tree
Showing 9 changed files with 214 additions and 230 deletions.
394 changes: 197 additions & 197 deletions .Rhistory

Large diffs are not rendered by default.

20 changes: 6 additions & 14 deletions SQL_production/TM_2022A_data.sql
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ select * into #tab3 from QRD.dbo.Table3_2022_10_31
If object_Id('tempDB..#tab4') is not null drop table #tab4
select * into #tab4 from QRD.dbo.Table4_2022_10_31
If object_Id('tempDB..#subj') is not null drop table #subj
--select distinct SUBLEVNO, SUBJ, MAPPING into #subj from [L3VA].[U2022].[QUAL_SUBJ_LOOKUP]
select distinct SUBJ, MAPPING into #subj from [L3VA].[U2022].[QUAL_SUBJ_LOOKUP]


If object_Id('tempDB..#raw_inst') is not null drop table #raw_inst
select * into #raw_inst from KS5_RESTRICTED.[Outputs].[RAW_Inst_POST16_2022A]
Expand Down Expand Up @@ -335,24 +334,22 @@ on a.GNUMBER = y.QUID and a.BRDSUBNO = y.Syllabus_Ref

--national, qualification and subject
If object_Id('tempDB..#nat_subj') is not null drop table #nat_subj
select Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
select Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, a.MAPPING as SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
count(PUPILID) as total_students
into #nat_subj
from #all_dat as a
left join #insti_data as d
on a.URN = d.URN
left join #subj e
on a.MAPPING = e.MAPPING --and a.SUBLEVNO = e.SUBLEVNO
where --TAB1618 = 1
((Qualification IN ('Other academic', 'Tech level', 'Applied general') and DISCS = 0) OR
(Qualification = 'A level' and (DISCS=0 OR (DISCS=1 AND DISC=1 AND DISCB=0))) OR
(Qualification = 'Technical certificate' and DISC_FULL = 0))
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
order by Qual_Description, SUBLEVNO, ASIZE, GSIZE, [Subject], GRADE_UPDATE, PRIOR_BAND

--create national data again but for AS so we can ignore usual discounting and implement AS only discounting
If object_Id('tempDB..#nat_subj_AS') is not null drop table #nat_subj_AS
select 'GCE AS level (All)' as Qual_Description, 699 as SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
select 'GCE AS level (All)' as Qual_Description, 699 as SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, a.MAPPING as SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
count(PUPILID) as total_students
into #nat_subj_AS
from (select *, ROW_NUMBER() OVER (PARTITION BY LAESTAB, PUPILID, MAPPING, SUBLEVNO
Expand All @@ -361,13 +358,8 @@ from #all_dat
where SUBLEVNO = 121) as a
left join #insti_data as d
on a.URN = d.URN
left join /*(
select distinct MAPPING, SUBJ
from*/ #subj
/*where SUBLEVNO IN (111, 121))*/ e
on a.MAPPING = e.MAPPING --and a.SUBLEVNO = e.SUBLEVNO
where MAINEXAM = 1 --TAB1618 = 1
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
order by Qual_Description, SUBLEVNO, ASIZE, GSIZE, [Subject], GRADE_UPDATE, PRIOR_BAND

-- bind the 2 tables together
Expand Down Expand Up @@ -411,7 +403,7 @@ where PRIOR_BAND is NULL
--drop table [KS5_STATISTICS_RESTRICTED].[TM_2022].[TM_data_2022A]

select *
into [KS5_STATISTICS_RESTRICTED].[TM_2022].[TM_data_2022A]
into [KS5_STATISTICS_RESTRICTED].[TM_2024].[TM_data_2022A]
from #TM_data


Expand Down
20 changes: 6 additions & 14 deletions SQL_production/TM_2023A_data.sql
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ select * into #tab3 from QRD.dbo.Table3_2023_10_31
If object_Id('tempDB..#tab4') is not null drop table #tab4
select * into #tab4 from QRD.dbo.Table4_2023_10_31
If object_Id('tempDB..#subj') is not null drop table #subj
--select distinct SUBLEVNO, SUBJ, MAPPING into #subj from [L3VA].[U2022].[QUAL_SUBJ_LOOKUP]
select distinct SUBJ, MAPPING into #subj from [L3VA].[U2023].[SUBJCODES_lookup]


If object_Id('tempDB..#raw_inst') is not null drop table #raw_inst
select * into #raw_inst from KS5_RESTRICTED.[Outputs].[RAW_Inst_POST16_202223A]
Expand Down Expand Up @@ -335,24 +334,22 @@ on a.GNUMBER = y.QUID and a.BRDSUBNO = y.Syllabus_Ref

--national, qualification and subject
If object_Id('tempDB..#nat_subj') is not null drop table #nat_subj
select Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
select Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, a.MAPPING as SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
count(PUPILID) as total_students
into #nat_subj
from #all_dat as a
left join #insti_data as d
on a.URN = d.URN
left join #subj e
on a.MAPPING = e.MAPPING --and a.SUBLEVNO = e.SUBLEVNO
where --TAB1618 = 1
((Qualification IN ('Other academic', 'Tech level', 'Applied general') and DISCS = 0) OR
(Qualification = 'A level' and (DISCS=0 OR (DISCS=1 AND DISC=1 AND DISCB=0))) OR
(Qualification = 'Technical certificate' and DISC_FULL = 0))
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
order by Qual_Description, SUBLEVNO, ASIZE, GSIZE, [Subject], GRADE_UPDATE, PRIOR_BAND

--create national data again but for AS so we can ignore usual discounting and implement AS only discounting
If object_Id('tempDB..#nat_subj_AS') is not null drop table #nat_subj_AS
select 'GCE AS level (All)' as Qual_Description, 699 as SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
select 'GCE AS level (All)' as Qual_Description, 699 as SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, a.MAPPING as SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE as GRADE, --Qualification,
count(PUPILID) as total_students
into #nat_subj_AS
from (select *, ROW_NUMBER() OVER (PARTITION BY LAESTAB, PUPILID, MAPPING, SUBLEVNO
Expand All @@ -361,13 +358,8 @@ from #all_dat
where SUBLEVNO = 121) as a
left join #insti_data as d
on a.URN = d.URN
left join /*(
select distinct MAPPING, SUBJ
from*/ #subj
/*where SUBLEVNO IN (111, 121))*/ e
on a.MAPPING = e.MAPPING --and a.SUBLEVNO = e.SUBLEVNO
where MAINEXAM = 1 --TAB1618 = 1
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, SUBJ, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
group by Qual_Description, a.SUBLEVNO, Potential_Level, ASIZE, GSIZE, a.MAPPING, [Subject], gradeStructure, PRIOR_BAND, GRADE_UPDATE --Qualification,
order by Qual_Description, SUBLEVNO, ASIZE, GSIZE, [Subject], GRADE_UPDATE, PRIOR_BAND

-- bind the 2 tables together
Expand Down Expand Up @@ -414,7 +406,7 @@ where [Subject] is NULL


select *
into [KS5_STATISTICS_RESTRICTED].[TM_2023].[TM_data_2023A]
into [KS5_STATISTICS_RESTRICTED].[TM_2024].[TM_data_2023A]
from #TM_data


Expand Down
6 changes: 3 additions & 3 deletions background_scripts/data_processing.R
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ source("./background_scripts/data_processing_func.R")
# ---- Things to change between runs ----
# -----------------------------------------------------------------------------------------------------------------------------

ancillary_save_path <- "//lonnetapp01/DSGA2/!!Secure Data/SFR/2023/KS5/February_2024/06_ancillary/"
ancillary_save_path <- "//lonnetapp01/DSGA2/!!Secure Data/SFR/2023/KS5/4 Dev/07_development/06_ancillary/"
current_year <- "2023A"

# -----------------------------------------------------------------------------------------------------------------------------
Expand All @@ -39,8 +39,8 @@ con <- DBI::dbConnect(odbc::odbc(), driver = "SQL Server", server = "VMT1PR-DHSQ

# Select data from SQL tables - need to add the current year for unamended run and change from U to A for the amended run

tm_data_raw_2023 <- tbl(con, sql("select * from [KS5_STATISTICS_RESTRICTED].[TM_2023].[TM_data_2023A]")) %>% collect()
tm_data_raw_2022 <- tbl(con, sql("select * from [KS5_STATISTICS_RESTRICTED].[TM_2022].[TM_data_2022A]")) %>% collect()
tm_data_raw_2023 <- tbl(con, sql("select * from [KS5_STATISTICS_RESTRICTED].[TM_2024].[TM_data_2023A]")) %>% collect()
tm_data_raw_2022 <- tbl(con, sql("select * from [KS5_STATISTICS_RESTRICTED].[TM_2024].[TM_data_2022A]")) %>% collect()

# disconnect
DBI::dbDisconnect(con)
Expand Down
4 changes: 2 additions & 2 deletions background_scripts/data_processing_func.R
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ TM_data_prod_func <- function(sql_data, ReportYear) {

tm_data <- sql_data %>%
filter(!(is.na(PRIOR_BAND)),
!(is.na(SUBJ))) %>%
bind_rows(tm_data_subj_na)
!(is.na(SUBJ))) #%>%
#bind_rows(tm_data_subj_na)

colSums(is.na(tm_data)) %>% as.data.frame()

Expand Down
Binary file modified data/all_student_numbers.rds
Binary file not shown.
Binary file modified data/all_student_percentages.rds
Binary file not shown.
Binary file modified data/grade_lookup.rds
Binary file not shown.
Binary file modified data/qual_lookup.rds
Binary file not shown.

0 comments on commit 11d0da5

Please sign in to comment.