diff --git a/.github/workflows/tests_metadata.yml b/.github/workflows/tests_metadata.yml index 556ab7b..a2fe8e1 100644 --- a/.github/workflows/tests_metadata.yml +++ b/.github/workflows/tests_metadata.yml @@ -17,6 +17,15 @@ jobs: image: nycplanning/dev:latest steps: - uses: actions/checkout@v4 + with: + repository: NYCPlanning/product-metadata + path: product-metadata + - uses: actions/checkout@v4 + with: + repository: NYCPlanning/data-engineering + path: data-engineering - name: Validate All Files - # run: python -m dcpy.cli - run: echo "hi" + run: cd data-engineering; python -m dcpy.cli lifecycle scripts product_metadata validate_repo ../product-metadata + env: + RECIPES_BUCKET: edm-recipes + PUBLISHING_BUCKET: edm-publishing diff --git a/metadata.yml b/metadata.yml new file mode 100644 index 0000000..d6300c2 --- /dev/null +++ b/metadata.yml @@ -0,0 +1,27 @@ +products: +- building_elevation_and_subgrade +- census_equivalency_tract_table +- colp +- community_development_block_grant_cdbg_eligibility_by_census_tract +- cpdb +- dcm +- designated_areas_m_districts_appendix_j +- directory_of_land_use_application_fees +- e_designations +- facilities +- fresh_zoning_boundary +- inclusionary_housing_designated_areas +- lion +- lower_density_growth_management_areas +- mandatory_inclusionary_housing +- new_york_city_neighborhood_name_centroid +- pluto +- pops +- projected_sea_level_rise +- template_db +- transit_zones +- waterfront_public_access_areas +- waterfront_revitalization_program +- zap +- zoning +- ztl diff --git a/products/building_elevation_and_subgrade/metadata.yml b/products/building_elevation_and_subgrade/metadata.yml new file mode 100644 index 0000000..ef6b252 --- /dev/null +++ b/products/building_elevation_and_subgrade/metadata.yml @@ -0,0 +1,4 @@ +id: building_elevation_and_subgrade + +datasets: + - building_elevation_and_subgrade diff --git a/products/census_equivalency_tract_table/metadata.yml b/products/census_equivalency_tract_table/metadata.yml new file mode 100644 index 0000000..93edc86 --- /dev/null +++ b/products/census_equivalency_tract_table/metadata.yml @@ -0,0 +1,5 @@ +id: census_equivalency_tract_table + +datasets: + - 2010_census_tract_to_neighborhood_tabulation_area_equivalency_table + - 2020_census_tracts_to_2020_ntas_and_cdtas_equivalency diff --git a/products/colp/metadata.yml b/products/colp/metadata.yml new file mode 100644 index 0000000..8d087d4 --- /dev/null +++ b/products/colp/metadata.yml @@ -0,0 +1,4 @@ +id: colp + +datasets: +- colp diff --git a/products/community_development_block_grant_cdbg_eligibility_by_census_tract/metadata.yml b/products/community_development_block_grant_cdbg_eligibility_by_census_tract/metadata.yml new file mode 100644 index 0000000..969d144 --- /dev/null +++ b/products/community_development_block_grant_cdbg_eligibility_by_census_tract/metadata.yml @@ -0,0 +1,4 @@ +id: community_development_block_grant_cdbg_eligibility_by_census_tract + +datasets: +- community_development_block_grant_cdbg_eligibility_by_census_tract diff --git a/products/cpdb/metadata.yml b/products/cpdb/metadata.yml new file mode 100644 index 0000000..7d84695 --- /dev/null +++ b/products/cpdb/metadata.yml @@ -0,0 +1,5 @@ +id: cpdb + +datasets: +- commitments +- projects diff --git a/products/dcm/city_map_alterations/metadata.yml b/products/dcm/city_map_alterations/metadata.yml index 8e2a9dd..c6bd14e 100644 --- a/products/dcm/city_map_alterations/metadata.yml +++ b/products/dcm/city_map_alterations/metadata.yml @@ -106,7 +106,7 @@ columns: data_type: text - id: effect_dt name: Effect_DT - data_type: date + data_type: text description: Indicates effective date of map alteration on Alteration map. - id: bp_num name: BP_NUM @@ -124,7 +124,7 @@ columns: effective. - id: cert_date name: Cert_Date - data_type: date + data_type: text description: The date the mapping application was certified by City Planning. - id: chg_type name: CHG_TYPE diff --git a/products/dcm/dcm/metadata.yml b/products/dcm/dcm/metadata.yml index 403f60a..4c1d82b 100644 --- a/products/dcm/dcm/metadata.yml +++ b/products/dcm/dcm/metadata.yml @@ -119,7 +119,7 @@ columns: data_type: text - id: edit_date name: Edit_Date - data_type: date + data_type: text description: The date the record was last edited. - id: the_geom name: the_geom diff --git a/products/dcm/metadata.yml b/products/dcm/metadata.yml new file mode 100644 index 0000000..577e17b --- /dev/null +++ b/products/dcm/metadata.yml @@ -0,0 +1,9 @@ +id: dcm + +datasets: +- dcm +- street_center_line +- city_map_alterations +- street_name_changes +- arterials_major_streets +- fgdb diff --git a/products/dcm/street_center_line/metadata.yml b/products/dcm/street_center_line/metadata.yml index af29034..ed990ea 100644 --- a/products/dcm/street_center_line/metadata.yml +++ b/products/dcm/street_center_line/metadata.yml @@ -185,7 +185,7 @@ columns: description: marginal street, wharf or place - id: edit_date name: Edit_Date - data_type: date + data_type: text description: The date the record was last edited. - id: the_geom name: the_geom diff --git a/products/designated_areas_m_districts_appendix_j/metadata.yml b/products/designated_areas_m_districts_appendix_j/metadata.yml new file mode 100644 index 0000000..deba422 --- /dev/null +++ b/products/designated_areas_m_districts_appendix_j/metadata.yml @@ -0,0 +1,4 @@ +id: designated_areas_m_districts_appendix_j + +datasets: +- designated_areas_m_districts_appendix_j diff --git a/products/directory_of_land_use_application_fees/metadata.yml b/products/directory_of_land_use_application_fees/metadata.yml new file mode 100644 index 0000000..b634cb6 --- /dev/null +++ b/products/directory_of_land_use_application_fees/metadata.yml @@ -0,0 +1,4 @@ +id: directory_of_land_use_application_fees + +datasets: +- directory_of_land_use_application_fees diff --git a/products/e_designations/metadata.yml b/products/e_designations/metadata.yml new file mode 100644 index 0000000..0f60961 --- /dev/null +++ b/products/e_designations/metadata.yml @@ -0,0 +1,4 @@ +id: e_designations + +datasets: +- e_designations diff --git a/products/facilities/facilities/metadata.yml b/products/facilities/facilities/metadata.yml index ff9e0a8..b797da0 100644 --- a/products/facilities/facilities/metadata.yml +++ b/products/facilities/facilities/metadata.yml @@ -41,122 +41,122 @@ destinations: custom: destination_use: attachment - id: shapefile - dataset_overrides: - overridden_columns: - - id: UID - custom: - api_name: uid - - id: FACNAME - custom: - api_name: facname - - id: ADDRESSNUM - custom: - api_name: addressnum - - id: STREETNAME - custom: - api_name: streetname - - id: ADDRESS - custom: - api_name: address - - id: CITY - custom: - api_name: city - - id: BORO - custom: - api_name: boro - - id: BOROCODE - custom: - api_name: borocode - - id: ZIPCODE - custom: - api_name: zipcode - - id: LATITUDE - custom: - api_name: latitude - - id: LONGITUDE - custom: - api_name: longitude - - id: XCOORD - custom: - api_name: xcoord - - id: YCOORD - custom: - api_name: ycoord - - id: BIN - custom: - api_name: bin - - id: BBL - custom: - api_name: bbl - - id: CD - custom: - api_name: cd - - id: COUNCIL - custom: - api_name: council - - id: CT2010 - custom: - api_name: ct2010 - - id: CT2020 - custom: - api_name: ct2020 - - id: NTA2010 - custom: - api_name: nta2010 - - id: NTA2020 - custom: - api_name: nta2020 - - id: FACGROUP - custom: - api_name: facgroup - - id: FACSUBGRP - custom: - api_name: facsubgrp - - id: FACTYPE - custom: - api_name: factype - - id: CAPACITY - custom: - api_name: capacity - - id: CAPTYPE - custom: - api_name: captype - - id: OPTYPE - custom: - api_name: optype - - id: OPNAME - custom: - api_name: opname - - id: OPABBREV - custom: - api_name: opabbrev - - id: OVERLEVEL - custom: - api_name: overlevel - - id: OVERAGENCY - custom: - api_name: overagency - - id: OVERABBREV - custom: - api_name: overabbrev - - id: DATASOURCE - custom: - api_name: datasource - - id: FACDOMAIN - custom: - api_name: facdomain - - id: SCHOOLDIST - custom: - api_name: schooldist - - id: POLICEPRCT - custom: - api_name: policeprct - - id: SERVAREA - custom: - api_name: servarea - - id: geometry - custom: - api_name: geometry + # dataset_overrides: + # overridden_columns: + # - id: UID + # custom: + # api_name: uid + # - id: FACNAME + # custom: + # api_name: facname + # - id: ADDRESSNUM + # custom: + # api_name: addressnum + # - id: STREETNAME + # custom: + # api_name: streetname + # - id: ADDRESS + # custom: + # api_name: address + # - id: CITY + # custom: + # api_name: city + # - id: BORO + # custom: + # api_name: boro + # - id: BOROCODE + # custom: + # api_name: borocode + # - id: ZIPCODE + # custom: + # api_name: zipcode + # - id: LATITUDE + # custom: + # api_name: latitude + # - id: LONGITUDE + # custom: + # api_name: longitude + # - id: XCOORD + # custom: + # api_name: xcoord + # - id: YCOORD + # custom: + # api_name: ycoord + # - id: BIN + # custom: + # api_name: bin + # - id: BBL + # custom: + # api_name: bbl + # - id: CD + # custom: + # api_name: cd + # - id: COUNCIL + # custom: + # api_name: council + # - id: CT2010 + # custom: + # api_name: ct2010 + # - id: CT2020 + # custom: + # api_name: ct2020 + # - id: NTA2010 + # custom: + # api_name: nta2010 + # - id: NTA2020 + # custom: + # api_name: nta2020 + # - id: FACGROUP + # custom: + # api_name: facgroup + # - id: FACSUBGRP + # custom: + # api_name: facsubgrp + # - id: FACTYPE + # custom: + # api_name: factype + # - id: CAPACITY + # custom: + # api_name: capacity + # - id: CAPTYPE + # custom: + # api_name: captype + # - id: OPTYPE + # custom: + # api_name: optype + # - id: OPNAME + # custom: + # api_name: opname + # - id: OPABBREV + # custom: + # api_name: opabbrev + # - id: OVERLEVEL + # custom: + # api_name: overlevel + # - id: OVERAGENCY + # custom: + # api_name: overagency + # - id: OVERABBREV + # custom: + # api_name: overabbrev + # - id: DATASOURCE + # custom: + # api_name: datasource + # - id: FACDOMAIN + # custom: + # api_name: facdomain + # - id: SCHOOLDIST + # custom: + # api_name: schooldist + # - id: POLICEPRCT + # custom: + # api_name: policeprct + # - id: SERVAREA + # custom: + # api_name: servarea + # - id: geometry + # custom: + # api_name: geometry custom: destination_use: dataset_file custom: diff --git a/products/facilities/metadata.yml b/products/facilities/metadata.yml new file mode 100644 index 0000000..eb092a8 --- /dev/null +++ b/products/facilities/metadata.yml @@ -0,0 +1,4 @@ +id: facilities + +datasets: +- facilities diff --git a/products/fresh_zoning_boundary/metadata.yml b/products/fresh_zoning_boundary/metadata.yml new file mode 100644 index 0000000..0981225 --- /dev/null +++ b/products/fresh_zoning_boundary/metadata.yml @@ -0,0 +1,4 @@ +id: fresh_zoning_boundary + +datasets: +- fresh_zoning_boundary diff --git a/products/inclusionary_housing_designated_areas/metadata.yml b/products/inclusionary_housing_designated_areas/metadata.yml new file mode 100644 index 0000000..1263bd2 --- /dev/null +++ b/products/inclusionary_housing_designated_areas/metadata.yml @@ -0,0 +1,4 @@ +id: inclusionary_housing_designated_areas + +datasets: +- inclusionary_housing_designated_areas diff --git a/products/lower_density_growth_management_areas/metadata.yml b/products/lower_density_growth_management_areas/metadata.yml new file mode 100644 index 0000000..086ed63 --- /dev/null +++ b/products/lower_density_growth_management_areas/metadata.yml @@ -0,0 +1,4 @@ +id: lower_density_growth_management_areas + +datasets: +- lower_density_growth_management_areas diff --git a/products/mandatory_inclusionary_housing/metadata.yml b/products/mandatory_inclusionary_housing/metadata.yml new file mode 100644 index 0000000..d56c9ee --- /dev/null +++ b/products/mandatory_inclusionary_housing/metadata.yml @@ -0,0 +1,4 @@ +id: mandatory_inclusionary_housing + +datasets: +- mandatory_inclusionary_housing diff --git a/products/new_york_city_neighborhood_name_centroid/metadata.yml b/products/new_york_city_neighborhood_name_centroid/metadata.yml new file mode 100644 index 0000000..3c4e932 --- /dev/null +++ b/products/new_york_city_neighborhood_name_centroid/metadata.yml @@ -0,0 +1,4 @@ +id: new_york_city_neighborhood_name_centroid + +datasets: +- new_york_city_neighborhood_name_centroid diff --git a/products/pluto/change_file/metadata.yml b/products/pluto/change_file/metadata.yml index bde6c65..e8a9ea0 100644 --- a/products/pluto/change_file/metadata.yml +++ b/products/pluto/change_file/metadata.yml @@ -37,7 +37,7 @@ destinations: - id: socrata type: socrata files: - - id: pluto_removed_records.csv + - id: pluto_removed_records custom: destination_use: attachment - id: pluto_changes_applied @@ -48,7 +48,7 @@ destinations: - id: socrata_test type: socrata files: - - id: pluto_removed_records.csv + - id: pluto_removed_records custom: destination_use: attachment - id: pluto_changes_applied diff --git a/products/pluto/metadata.yml b/products/pluto/metadata.yml new file mode 100644 index 0000000..b87b1c7 --- /dev/null +++ b/products/pluto/metadata.yml @@ -0,0 +1,5 @@ +id: pluto + +datasets: +- change_file +- pluto diff --git a/products/pops/metadata.yml b/products/pops/metadata.yml new file mode 100644 index 0000000..f39b43e --- /dev/null +++ b/products/pops/metadata.yml @@ -0,0 +1,4 @@ +id: pops + +datasets: +- pops diff --git a/products/projected_sea_level_rise/metadata.yml b/products/projected_sea_level_rise/metadata.yml new file mode 100644 index 0000000..36c7350 --- /dev/null +++ b/products/projected_sea_level_rise/metadata.yml @@ -0,0 +1,4 @@ +id: projected_sea_level_rise + +datasets: +- projected_sea_level_rise diff --git a/products/template_db/metadata.yml b/products/template_db/metadata.yml new file mode 100644 index 0000000..d3c42eb --- /dev/null +++ b/products/template_db/metadata.yml @@ -0,0 +1,4 @@ +id: template_db + +datasets: +- template_db diff --git a/products/template_db/template_db/metadata.yml b/products/template_db/template_db/metadata.yml index aa23fee..cb90bb8 100644 --- a/products/template_db/template_db/metadata.yml +++ b/products/template_db/template_db/metadata.yml @@ -23,11 +23,6 @@ destinations: custom: destination_use: attachment - id: shapefile_points - dataset_overrides: - overridden_columns: - - id: the_geom - custom: - api_name: wkb_geometry custom: destination_use: dataset_file custom: diff --git a/products/transit_zones/metadata.yml b/products/transit_zones/metadata.yml new file mode 100644 index 0000000..e0469fd --- /dev/null +++ b/products/transit_zones/metadata.yml @@ -0,0 +1,4 @@ +id: transit_zones + +datasets: +- transit_zones diff --git a/products/waterfront_public_access_areas/metadata.yml b/products/waterfront_public_access_areas/metadata.yml index 5b39939..aaae79a 100644 --- a/products/waterfront_public_access_areas/metadata.yml +++ b/products/waterfront_public_access_areas/metadata.yml @@ -1,75 +1,10 @@ -id: waterfront_access_map_combined - -attributes: - description: - The Waterfront Access Map (WAM) data contains information about waterfront - areas accessible to the public that are either publicly owned or on private property. - The Waterfront Public Access Areas (WPAA)s data contains privately owned waterfront - zoning lots where publicly accessible open space is provided to and along the - shoreline for public enjoyment. The Publicly Owned Waterfront data contains City, - State, and Federally owned public parks and facilities that provide waterfront - parkland and open space for public enjoyment. Data for WPAA footprints and access - points, human powered boat launches and saltwater fishing access are also included. - All Waterfront Access Map (WAM) datasets are featured on the - NYC DCP Waterfront Access Map - display_name: "Waterfront Access Map Data: Geodatabase" - each_row_is_a: see metadata - tags: - - waterfront public access areas - - wpaa - - open space - - parks - - footprints - - access points - - publicly owned waterfront - - human powered boat launches - - boating - - saltwater fishing - - waterfront access map - - wam - - publicly accessible waterfront - - shore public access - - city planning - - dcp - - nyc - -assembly: [] - -custom: {} - -destinations: - - id: socrata_prod - type: socrata - files: - - id: fgdb - custom: - destination_use: dataset_file - - id: paws_user_guide - custom: - destination_use: attachment - - id: meta_zip - custom: - destination_use: attachment - custom: - four_four: sp9a-cd2a - is_unparsed_dataset: true - - id: bytes - type: bytes - files: - - id: fgdb - custom: - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycwpaas_{{ version }}fgdb.zip - -files: - - file: - id: fgdb - filename: nycwpaas.zip - type: fgdb - - file: - id: paws_user_guide - filename: paws_user_guide.pdf - - file: - id: meta_zip - filename: nycwpaas_meta.zip - -columns: [] +id: waterfront_public_access_areas + +datasets: +- hpb_launches +- nyc_saltwaterfishingsites +- publicly_owned_waterfront +- waterfront_public_access_areas +- wpaas +- wpaas_accesspoints +- wpaas_footprints diff --git a/products/waterfront_public_access_areas/waterfront_public_access_areas/metadata.yml b/products/waterfront_public_access_areas/waterfront_public_access_areas/metadata.yml new file mode 100644 index 0000000..5b39939 --- /dev/null +++ b/products/waterfront_public_access_areas/waterfront_public_access_areas/metadata.yml @@ -0,0 +1,75 @@ +id: waterfront_access_map_combined + +attributes: + description: + The Waterfront Access Map (WAM) data contains information about waterfront + areas accessible to the public that are either publicly owned or on private property. + The Waterfront Public Access Areas (WPAA)s data contains privately owned waterfront + zoning lots where publicly accessible open space is provided to and along the + shoreline for public enjoyment. The Publicly Owned Waterfront data contains City, + State, and Federally owned public parks and facilities that provide waterfront + parkland and open space for public enjoyment. Data for WPAA footprints and access + points, human powered boat launches and saltwater fishing access are also included. + All Waterfront Access Map (WAM) datasets are featured on the + NYC DCP Waterfront Access Map + display_name: "Waterfront Access Map Data: Geodatabase" + each_row_is_a: see metadata + tags: + - waterfront public access areas + - wpaa + - open space + - parks + - footprints + - access points + - publicly owned waterfront + - human powered boat launches + - boating + - saltwater fishing + - waterfront access map + - wam + - publicly accessible waterfront + - shore public access + - city planning + - dcp + - nyc + +assembly: [] + +custom: {} + +destinations: + - id: socrata_prod + type: socrata + files: + - id: fgdb + custom: + destination_use: dataset_file + - id: paws_user_guide + custom: + destination_use: attachment + - id: meta_zip + custom: + destination_use: attachment + custom: + four_four: sp9a-cd2a + is_unparsed_dataset: true + - id: bytes + type: bytes + files: + - id: fgdb + custom: + url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycwpaas_{{ version }}fgdb.zip + +files: + - file: + id: fgdb + filename: nycwpaas.zip + type: fgdb + - file: + id: paws_user_guide + filename: paws_user_guide.pdf + - file: + id: meta_zip + filename: nycwpaas_meta.zip + +columns: [] diff --git a/products/waterfront_revitalization_program/metadata.yml b/products/waterfront_revitalization_program/metadata.yml new file mode 100644 index 0000000..21b281b --- /dev/null +++ b/products/waterfront_revitalization_program/metadata.yml @@ -0,0 +1,4 @@ +id: waterfront_revitalization_program + +datasets: +- waterfront_revitalization_program diff --git a/products/zap/metadata.yml b/products/zap/metadata.yml new file mode 100644 index 0000000..42b5267 --- /dev/null +++ b/products/zap/metadata.yml @@ -0,0 +1,5 @@ +id: zap + +datasets: +- projects +- bbls diff --git a/products/zoning/metadata.yml b/products/zoning/metadata.yml new file mode 100644 index 0000000..e555ca7 --- /dev/null +++ b/products/zoning/metadata.yml @@ -0,0 +1,14 @@ +id: zoning + +attributes: + publishing_frequency: Monthly + +datasets: +- commercial_overlay_district +- zoning_districts +- zoning_map_amendments +- special_purpose_districts_subdistricts +- zoning_features +- limited_height_districts +- special_purpose_districts +- georeferenced_nyc_zoning_maps diff --git a/products/ztl/metadata.yml b/products/ztl/metadata.yml new file mode 100644 index 0000000..5d7678c --- /dev/null +++ b/products/ztl/metadata.yml @@ -0,0 +1,4 @@ +id: ztl + +datasets: +- ztl