Sample data and README file for NCES STEM Degrees import

statvar_imports/NCES/STEM_Degrees/README.md

@@ -0,0 +1,20 @@
+# NCES - Number and percentage distribution of science, technology, engineering, and mathematics (STEM)
+
+- source: hhttps://nces.ed.gov/programs/digest/d22/tables/dt22_318.45.asp
+
+- how to download data: Statvar processor downloads the xls file by using --data_url=https://nces.ed.gov/programs/digest/d23/tables/xls/tabn318.45.xlsx
+
+- type of place: Country
+
+- statvars: Education
+
+- years: 1990 to 2022
+
+
+### How to run:
+
+`python3 stat_var_processor.py --data_url="https://nces.ed.gov/programs/digest/d23/tables/xls/tabn318.45.xlsx" --input_data=source_data/input.xls --pv_map=statvar_imports/NCES/STEM_Degrees/pv_map/STEM_pv_map.csv --config=statvar_imports/NCES/STEM_Degrees/STEM_metadata.csv --output_path=--output_path=<filepath/filename>`
+
+#### Example
+`python3 stat_var_processor.py --data_url=https://nces.ed.gov/programs/digest/d23/tables/xls/tabn313.30.xlsx --input_data=/statvar_imports/NCES/STEM_Degrees/test_data/sample_input/input.xls --pv_map=/statvar_imports/NCES/STEM_Degrees/pv_map/STEM_pv_map.csv --config=/statvar_imports/NCES/STEM_Degrees/STEM_metadata.csv --output_path=/statvar_imports/NCES/STEM_Degrees/test_data/sample_output/nces_stem`
+

statvar_imports/NCES/STEM_Degrees/STEM_metadata.csv

@@ -0,0 +1,5 @@
+param,value
+output_columns,"observationAbout,observationDate,value,variableMeasured,unit,scalingFactor,measurementMethod,observationPeriod"
+url,https://nces.ed.gov/ipeds/Search?query=percentage%20of%20STEM%20higher%20degree%20holders%20are%20Black%20women&query2=percentage%20of%20STEM%20higher%20degree%20holders%20are%20Black%20women&resultType=all&page=1&sortBy=date_desc&overlayDigestTableId=202325
+drop_statvars_without_svobs,0
+ignore_rows,"[1,5]"

statvar_imports/NCES/STEM_Degrees/pv_map/STEM_pv_map.csv

@@ -0,0 +1,53 @@
+key,p1,v1,p2,v2,p3,v3,p4,v4,,,,,,
+Total,#populationType,Person,measuredProperty,count,statType,measuredValue,observationAbout,country/USA,gender,"""""",,,,
+Males,gender,Male,,,,,,,,,,,,
+Females,gender,Female,,,,,,,,,,,,
+Sex and year,#Regex,(?P<observationDate>[0-9]{4}),,,,,,,,,,,,
+AllRaces,value,{Number},race,"""""",ethnicity,"""""",,,,,,,,
+White,race,WhiteAlone,value,{Number},ethnicity,"""""",,,,,,,,
+Black,race,Black,value,{Number},ethnicity,"""""",,,,,,,,
+Hispanic,ethnicity,HispanicOrLatino,value,{Number},race,"""""",,,,,,,,
+Asian/Pacific Islander,race,AsianOrPacificIslander,value,{Number},ethnicity,"""""",,,,,,,,
+Asian/Pacific Islander:Total,race,AsianOrPacificIslander,value,{Number},ethnicity,"""""",,,,,,,,
+Both,race,AsianOrPacificIslander,value,{Number},ethnicity,"""""",,,,,,,,
+Asian,race,Asian,value,{Number},ethnicity,"""""",,,,,,,,
+Pacific Islander,race,HawaiianNativeOrPacificIslander,value,{Number},ethnicity,"""""",#MergedCell,FALSE,,,,,,
+Asian/Pacific Islander:Pacific Islander,race,HawaiianNativeOrPacificIslander,value,{Number},ethnicity,"""""",,,,,,,,
+American Indian/Alaska Native,race,AmericanIndianOrAlaskaNativeAlone,value,{Number},ethnicity,"""""",,,,,,,,
+Two or more races,race,TwoOrMoreRaces,value,{Number},ethnicity,"""""",,,,,,,,
+Nonresident,race ,"""""",value,{Number},ethnicity,"""""",residentStatus,NonUSResident,,,,,,
+"Number of STEM degrees/certificates conferred to U.S. citizens, permanent residents, and nonresidents",#populationType,Person,measuredProperty,count,statType,measuredValue,observationAbout,country/USA,bachelorsDegreeMajor,Science__Technology__Engineering__Mathematics,gender,"""""",value,{Number}
+all levels,educationalAttainment,CollegeGraduate,populationType,Person,,,,,,,,,,
+"Total, all levels of degrees/certificates",educationalAttainment,CollegeGraduate,populationType,Person,,,,,,,,,,
+Certificates below the associate's degree level,educationalAttainment,9ThTo12ThGradeNoDiploma,populationType,Person,,,,,,,,,,
+Certificates below the baccalaureate,educationalAttainment,9ThTo12ThGradeNoDiploma,populationType,Person,,,,,,,,,,
+,,,,,,,,,,,,,,
+Associate's degrees,educationalAttainment,AssociatesDegree,populationType,Person,,,,,,,,,,
+Bachelor's degrees,educationalAttainment,BachelorsDegree,populationType,Person,,,,,,,,,,
+Master's degrees,educationalAttainment,MastersDegree,populationType,Person,,,,,,,,,,
+Doctor's degrees,educationalAttainment,DoctorateDegree,populationType,Person,,,,,,,,,,
+Percentage distribution of STEM degrees/certificates conferred to U.S. citizens and permanent residents,unit,Percent,scalingFactor,100,bachelorsDegreeMajor,Science__Technology__Engineering__Mathematics,measurementDenominator,"-race,-ethnicity,-bachelorsDegreeMajor",value,{Number},,,,
+Column Number,#ignore,skip row with column numbers,,,,,,,,,,,,
+Percentage distribution of STEM degrees/certificates conferred to U.S. citizens and permanent residents:100,#ignore,To ignore the values with same numerator and denominator,,,,,,,,,,,,
+"Number of STEM degrees/certificates conferred to U.S. citizens, permanent residents, and nonresidents:Total",value,{Number},,,,,,,,,,,,
+Academic years,observationDate,"""""",,,,,,,,,,,,
+Number and percentage distribution,observationDate,"""""",,,,,,,,,,,,
+2012-13,observationDate,2012,,,,,,,,,,,,
+2013-14,observationDate,2013,,,,,,,,,,,,
+2014-15,observationDate,2014,,,,,,,,,,,,
+2015-16,observationDate,2015,,,,,,,,,,,,
+2016-17,observationDate,2016,,,,,,,,,,,,
+2017-18,observationDate,2017,,,,,,,,,,,,
+2018-19,observationDate,2018,,,,,,,,,,,,
+2019-20,observationDate,2019,,,,,,,,,,,,
+2020-21,observationDate,2020,,,,,,,,,,,,
+2021-22,observationDate,2021,,,,,,,,,,,,
+2022-23,observationDate,2022,,,,,,,,,,,,
+2023-24,observationDate,2023,,,,,,,,,,,,
+2024-25,observationDate,2024,,,,,,,,,,,,
+100,#ignore,ignorew,,,,,,,,,,,,
+2025-26,observationDate,2025,,,,,,,,,,,,
+2026-27,observationDate,2026,,,,,,,,,,,,
+2027-28,observationDate,2027,,,,,,,,,,,,
+2028-29,observationDate,2028,,,,,,,,,,,,
+2029-30,observationDate,2029,,,,,,,,,,,,