From c744c27d678578943596eb9e435dcfe6e8873471 Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Sat, 14 May 2022 13:28:16 -0700 Subject: [PATCH 01/10] For reference: specifications from the ObsCore v1.1 standard, tables 6 and 7 --- yml/ivoa/ObsCore-v1.1-mandatory.csv | 31 ++++++++++++++++++ yml/ivoa/ObsCore-v1.1-optional.csv | 31 ++++++++++++++++++ yml/ivoa/README.md | 51 +++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 yml/ivoa/ObsCore-v1.1-mandatory.csv create mode 100644 yml/ivoa/ObsCore-v1.1-optional.csv create mode 100644 yml/ivoa/README.md diff --git a/yml/ivoa/ObsCore-v1.1-mandatory.csv b/yml/ivoa/ObsCore-v1.1-mandatory.csv new file mode 100644 index 00000000..3f991c1b --- /dev/null +++ b/yml/ivoa/ObsCore-v1.1-mandatory.csv @@ -0,0 +1,31 @@ +Column Name,Datatype,Size,Units,ObsCoreDM Utype,UCD,Principal,Index,Std +dataproduct_type,adql:VARCHAR,TBD,NULL,ObsDataset.dataProductType,meta.id,1,TBD,1 +calib_level,adql:INTEGER,NULL,NULL,ObsDataset.calibLevel,meta.code;obs.calib,1,TBD,1 +obs_collection,adql:VARCHAR,TBD,NULL,DataID.collection,meta.id,1,TBD,1 +obs_id,adql:VARCHAR,TBD,NULL,DataID.observationID,meta.id,1,TBD,1 +obs_publisher_did,adql:VARCHAR,TBD,NULL,Curation.publisherDID,meta.ref.uri;meta.curation,1,TBD,1 +access_url,adql:CLOB,NULL,NULL,Access.reference,meta.ref.url,1,0,1 +access_format,adql:VARCHAR,NULL,NULL,Access.format,meta.code.mime,1,0,1 +access_estsize,adql:BIGINT,NULL,kbyte,Access.size,phys.size;meta.file,1,0,1 +target_name,adql:VARCHAR,TBD,NULL,Target.name,meta.id;src,1,0,1 +s_ra,adql:DOUBLE,NULL,deg,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1,pos.eq.ra,1,0,1 +s_dec,adql:DOUBLE,NULL,deg,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2,pos.eq.dec,1,0,1 +s_fov,adql:DOUBLE,NULL,deg,Char.SpatialAxis.Coverage.Bounds.Extent.diameter,phys.angSize;instr.fov,1,0,1 +s_region,adql:REGION,NULL,,Char.SpatialAxis.Coverage.Support.Area,pos.outline;obs.field,1,0,1 +s_resolution,adql:DOUBLE,NULL,arcsec,Char.SpatialAxis.Resolution.Refval.value,pos.angResolution,1,TBD,1 +s_xel1,adql:BIGINT,NULL,NULL,Char.SpatialAxis.numBins1,meta.number,1,TBD,1 +s_xel2,adql:BIGINT,NULL,NULL,Char.SpatialAxis.numBins2,meta.number,1,TBD,1 +t_min,adql:DOUBLE,NULL,d,Char.TimeAxis.Coverage.Bounds.Limits.StartTime,time.start;obs.exposure,1,0,1 +t_max,adql:DOUBLE,NULL,d,Char.TimeAxis.Coverage.Bounds.Limits.StopTime,time.end;obs.exposure,1,0,1 +t_exptime,adql:DOUBLE,NULL,s,Char.TimeAxis.Coverage.Support.Extent,time.duration;obs.exposure,1,TBD,1 +t_resolution,adql:DOUBLE,NULL,s,Char.TimeAxis.Resolution.Refval.value,time.resolution,1,0,1 +t_xel,adql: BIGINT,NULL,NULL,Char.TimeAxis.numBins,meta.number,1,TBD,1 +em_min,adql:DOUBLE,NULL,m,Char.SpectralAxis.Coverage.Bounds.Limits.LoLimit,em.wl;stat.min,1,0,1 +em_max,adql:DOUBLE,NULL,m,Char.SpectralAxis.Coverage.Bounds.Limits.HiLimit,em.wl;stat.max,1,0,1 +em_res_power,adql:DOUBLE,NULL,NULL,Char.SpectralAxis.Resolution.ResolPower.refVal,spect.resolution,1,TBD,1 +em_xel,adql: BIGINT,NULL,NULL,Char.SpectralAxis.numBins,meta.number,1,TBD,1 +o_ucd,adql:VARCHAR,TBD,NULL,Char.ObservableAxis.ucd,meta.ucd,1,0,1 +pol_states,adql:VARCHAR,TBD,NULL,Char.PolarizationAxis.stateList,meta.code;phys.polarization,1,0,1 +pol_xel,adql: BIGINT,NULL,NULL,Char.PolarizationAxis.numBins,meta.number,1,TBD,1 +facility_name,adql:VARCHAR,NULL,NULL,Provenance.ObsConfig.Facility.name,meta.id;instr.tel ,1,TBD,1 +Instrument_name,adql:VARCHAR,NULL ,NULL,Provenance.ObsConfig.Instrument.name,meta.id;instr,1,TBD,1 diff --git a/yml/ivoa/ObsCore-v1.1-optional.csv b/yml/ivoa/ObsCore-v1.1-optional.csv new file mode 100644 index 00000000..c0d7a358 --- /dev/null +++ b/yml/ivoa/ObsCore-v1.1-optional.csv @@ -0,0 +1,31 @@ +Column Name,Datatype,Size,Units,ObsCoreDM Utype,UCD,Principal,Index,Std +dataproduct_subtype,adql:VARCHAR,TBD,NULL,ObsDataset.dataProductSubtype,meta.id,1,TBD,1 +target_class,adql:VARCHAR,TBD,NULL,Target.class,src.class,1,TBD,1 +obs_creation_date,adql:TIMESTAMP,NULL,NULL,DataID.date,time;meta.dataset,1,TBD,1 +obs_creator_name,adql:VARCHAR,TBD,NULL,DataID.creator,meta.id,1,TBD,1 +obs_creator_did,adql:VARCHAR,TBD,NULL,DataID.creatorDID,meta.id,0,TBD,1 +obs_title,adql:VARCHAR,200,NULL,DataID.title,meta.title;obs,1,0,1 +publisher_id,adql:VARCHAR,TBD,NULL,Curation.publisherID,meta.ref.uri;meta.curation,1,TBD,1 +bib_reference,adql:VARCHAR,TBD,NULL,Curation.reference,meta.bib,0,0,1 +data_rights,adql:VARCHAR,NULL, NULL,Curation.rights,meta.code,0,0,1 +obs_release_date,adql:TIMESTAMP,NULL,NULL,Curation.releaseDate,time.release,1,0,1 +s_ucd,adql:VARCHAR,NULL,NULL,Char.SpatialAxis.ucd,meta.ucd,1,0,1 +s_unit,adql:VARCHAR,NULL,NULL,Char.SpatialAxis.unit,meta.unit,1,0,1 +s_resolution_min,adql:DOUBLE,NULL,arcsec,Char.SpatialAxis.Resolution.Bounds.Limits.LoLimit,pos.angResolution;stat.min,1,0,1 +s_resolution_max,adql:DOUBLE,NULL,arcsec,Char.SpatialAxis.Resolution.Bounds.Limits.HiLimit,pos.angResolution;stat.max,1,0,1 +s_pixel_scale,adql:DOUBLE,NULL,arcsec,Char.SpatialAxis.Sampling.RefVal.SamplingPeriod,phys.angSize;instr.pixel,1,0,1 +s_calib_status,adql:VARCHAR,NULL,NULL,Char.SpatialAxis.calibrationStatus,meta.code.qual,1,0,1 +s_stat_error,adql:DOUBLE,NULL,arcsec,Char.SpatialAxis.Accuracy.StatError.Refval.value,stat.error;pos.eq,0,0,1 +t_calib_status,adql:VARCHAR,NULL,NULL,Char.TimeAxis.calibrationStatus,meta.code.qual,0,0,1 +t_stat_error,adql:DOUBLE,NULL,s,Char.TimeAxis.Accuracy.StatError.Refval.value,stat.error;time,0,0,1 +em_ucd,adql:VARCHAR,NULL,NULL,Char.SpectralAxis.ucd,meta.ucd,1,TBD,1 +em_unit,adql:VARCHAR,NULL,NULL,Char.SpectralAxis.unit,meta.unit,1,0,1 +em_calib_status,adql:VARCHAR,NULL,NULL,Char.SpectralAxis.calibrationStatus,meta.code.qual,0,0,1 +em_res_power_min,adql:DOUBLE,NULL,NULL ,Char.SpectralAxis.Resolution.ResolPower.LoLimit,spect.resolution;stat.min,1,0,1 +em_res_power_max,adql:DOUBLE,NULL,NULL,Char.SpectralAxis.Resolution.ResolPower.HiLimit,spect.resolution;stat.max,1,0,1 +em_resolution,adql:DOUBLE,NULL,m,Char.SpectralAxis.Resolution.Refval.value,spect.resolution;stat.mean,1,0,1 +em_stat_error,adql:DOUBLE,NULL,m,Char.SpectralAxis.Accuracy.StatError.Refval.value,stat.error;em,0,0,1 +o_unit,adql:VARCHAR,NULL,NULL,Char.ObservableAxis.unit,meta.unit,1,0,1 +o_calib_status,adql:VARCHAR,NULL,NULL,Char.ObservableAxis.calibrationStatus,meta.code.qual,1,TBD,1 +o_stat_error,adql:DOUBLE,NULL,same units as the value of the o_unit attribute,Char.ObservableAxis.Accuracy.StatError.Refval.value,stat.error;phot.flux,0,0,1 +proposal_id,adql:VARCHAR,NULL,NULL,Provenance.Proposal.identifier,meta.id;obs.proposal,0,TBD,1 diff --git a/yml/ivoa/README.md b/yml/ivoa/README.md new file mode 100644 index 00000000..e310a96a --- /dev/null +++ b/yml/ivoa/README.md @@ -0,0 +1,51 @@ +Tools and Data Supporting IVOA Data Models +========================================== + +The directory `sdm_schemas/yml/ivoa` contains data and software supporting the +generation of Felis YAML for representations of IVOA data models. + +Initially this is limited to the ObsCore data model for observation metadata, +in its "ObsTAP" relational-database realization. +The Rubin Science Platform will provide ObsTAP, and eventually SIAv2, +observation-metadata services, and their output will be defined by a Felis +data model. + +To support that work and ensure that standards-compliant schema (TAP_SCHEMA) +metadata are supplied by the RSP services for that data model, this directory +contains CSV files with the metadata supplied by the IVOA ObsCore v1.1 (as of +this writing) standard, with published errata applied, and with certain very +minor typographical errors corrected. + +The Python script `make_obscore_nominal.py` uses that data, together with an +optional list of columns subsetting the full ObsCore data model, to emit Felis +YAML code defining that model. + +It is anticipated that the resulting YAML may need further editing, to add +RSP-specific extensions, or to customize the description text in the model for +Rubin-specific, and pedagogical, purposes. +Therefore, at present, the script cannot be used to build a fully automated +toolchain for the creation of the final Felis for data served by the RSP. + +CSV File Provenance +------------------- + +* The file `ObsCore-v1.1-descriptions.csv` was derived from Table 5, "Data model +summary" in Appendix B of the +[ObsCore v1.1 standard](https://www.ivoa.net/documents/ObsCore/20170509/index.html). +Several corrections from the +[published 2nd erratum](https://wiki.ivoa.net/twiki/bin/view/IVOA/ObsCore-1_1-Erratum-2) +to the standard were applied, followed by purely typographical corrections to ensure +consistency of spacing and case. +The provenance of these edits has been maintained as separate commits +in the file history. +* The files `ObsCore-v1.1-mandatory.csv` and `ObsCore-v1.1-optional.csv` were +derived from Table 6, "TAP_SCHEMA.columns values for the mandatory fields of +an ObsTAP table", and Table 7, "... for the optional fields for an ObsTAP table.", +respectively, in Appendix C of the standard. +Note that these two tables have the same layout in the standard and the CSV files +accordingly have the same schema. +They have been cross-checked for consistency against `ObsCore-v1.1-descriptions.csv`, +but that file has a different schema, like its underlying table in the standard. +Corrections from the +[published 1st erratum](https://wiki.ivoa.net/twiki/bin/view/IVOA/ObsCore-1_1-Erratum-1) +have been applied, as well as additional purely typographical corrections. From d048a0490942943a01636c4ba97ca86bc6c05b6f Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Mon, 16 May 2022 11:37:48 -0700 Subject: [PATCH 02/10] Added descriptions from Table 5 of ObsCore v1.1 --- yml/ivoa/ObsCore-v1.1-descriptions.csv | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 yml/ivoa/ObsCore-v1.1-descriptions.csv diff --git a/yml/ivoa/ObsCore-v1.1-descriptions.csv b/yml/ivoa/ObsCore-v1.1-descriptions.csv new file mode 100644 index 00000000..7ccf20c1 --- /dev/null +++ b/yml/ivoa/ObsCore-v1.1-descriptions.csv @@ -0,0 +1,61 @@ +Column Name,Utype,Unit,Type,Description,MAN +dataproduct_type,ObsDataset.dataProductType,unitless,enum string,Data product (file content) primary type,YES +dataproduct_subtype,ObsDataset.dataProductSubtype,unitless,string,Data product specific type,NO +calib_level,ObsDataset.calibLevel,unitless,"enum int","Calibration level of the observation: in {0, 1, 2, 3, 4}",YES +target_name,Target.name,unitless,string,Object of interest,YES +target_class,Target.class,unitless,string,Class of the Target object as in SSA,NO +obs_id,DataID.observationID,unitless,string,Internal ID given by the ObsTAP service,YES +obs_title,DataID.title,unitless,string,Brief description of dataset in free format,NO +obs_collection,DataID.collection,unitless,string,Name of the data collection,YES +obs_creation_date,DataID.date,unitless,date,Date when the dataset was created,NO +obs_creator_name,DataID.creator,unitless,string,Name of the creator of the data,NO +obs_creator_did,DataID.creatorDID,unitless,string,IVOA dataset identifier given by the creator,NO +obs_release_date,Curation.releaseDate,unitless,date,Observation release date (ISO 8601),NO +obs_publisher_did,Curation.publisherDID,unitless,string,ID for the Dataset given by the publisher,YES +publisher_id,Curation.publisherID,unitless,string,IVOA-ID for the Publisher,NO +bib_reference,Curation.reference,unitless,string,Service bibliographic reference,NO +data_rights,Curation.rights,unitless,enum string,Public/Secure/Proprietary/,NO +access_url,Access.reference,unitless,string,URL used to access dataset,YES +access_format,Access.format,unitless,string,Content format of the dataset,YES +access_estsize,Access.size,kbyte,int,Estimated size of dataset: in kilobytes,YES +s_ra,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1,deg,double,"Central Spatial Position in ICRS; Right ascension",YES +s_dec,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2,deg,double,"Central Spatial Position in ICRS; Declination",YES +s_fov,Char.SpatialAxis.Coverage.Bounds.Extent.diameter,deg,double,Estimated size of the covered region as the diameter of a containing circle,YES +s_region,Char.SpatialAxis.Coverage.Support.Area,,string,Sky region covered by the data product (expressed in ICRS frame),YES +s_resolution,Char.SpatialAxis.Resolution.refval.value,arcsec,double,Spatial resolution of data as FWHM of PSF,YES +s_xel1,Char.SpatialAxis.numBins1,unitless,integer,Number of elements along the first coordinate of the spatial axis,YES +s_xel2,Char.SpatialAxis.numBins2,unitless,integer,Number of elements along the second coordinate of the spatial axis,YES +s_ucd,Char.SpatialAxis.ucd,unitless,string,"UCD for the nature of the spatial axis (pos or u,v data)",NO +s_unit,Char.SpatialAxis.unit,unitless,string,Unit used for spatial axis,NO +s_resolution_min,Char.SpatialAxis.Resolution.Bounds.Limits.LoLimit,arcsec,double,Resolution min value on spatial axis (FHWM of PSF),NO +s_resolution_max,Char.SpatialAxis.Resolution.Bounds.Limits.HiLimit,arcsec,double,Resolution max value on spatial axis,NO +s_calib_status,Char.SpatialAxis.calibrationStatus,unitless,Enum string,Type of calibration along the spatial axis,NO +s_stat_error,Char.SpatialAxis.Accuracy.StatError.Refval.value,arcsec,double,Astrometric precision along the spatial axis,NO +s_pixel_scale,Char.SpatialAxis.Sampling.RefVal.SamplingPeriod,arcsec,double,Sampling period in world coordinate units along the spatial axis,NO +t_xel,Char.TimeAxis.numBins,unitless,integer,Number of elements along the time axis,YES +t_refpos,Char.TimeAxis.ReferencePosition,unitless,Enum string,"Time Axis Reference Position as defined in STC REC, Section 4.4.1.1.1",NO +t_min,Char.TimeAxis.Coverage.Bounds.Limits.StartTime,d,double,Start time in MJD,YES +t_max,Char.TimeAxis.Coverage.Bounds.Limits.StopTime,d,double,Stop time in MJD,YES +t_exptime,Char.TimeAxis.Coverage.Support.Extent,s,double,Total exposure time,YES +t_resolution,Char.TimeAxis.Resolution.Refval.valueResolution.Refval.value,s,double,Temporal resolution FWHM,YES +t_calib_status,Char.TimeAxis.calibrationStatus,unitless,Enum string,Type of time coordinate calibration,NO +t_stat_error,Char.TimeAxis.Accuracy.StatError.Refval.value,s,double,Time coord statistical error,NO +em_xel,Char.SpectralAxis.numBins,unitless,integer,Number of elements along the spectral axis,YES +em_ucd,Char.SpectralAxis.ucd,unitless,string,Nature of the spectral axis,NO +em_unit,Char.SpectralAxis.unit,unitless,string,Units along the spectral axis,NO +em_calib_status,Char.SpectralAxis.calibrationStatus,unitless,Enum string,Type of spectral coord calibration,NO +em_min,Char.SpectralAxis.Coverage.Bounds.Limits.LoLimit,m,double,start in spectral coordinates,YES +em_max,Char.SpectralAxis.Coverage.Bounds.Limits.HiLimit,m,double,stop in spectral coordinates,YES +em_res_power,Char.SpectralAxis.Resolution.ResolPower.refVal,unitless,double,Value of the resolving power along the spectral axis (R),YES +em_res_power_min,Char.SpectralAxis.Resolution.ResolPower.LoLimit,unitless,double,Resolving power min value on spectral axis,NO +em_res_power_max,Char.SpectralAxis.Resolution.ResolPower.HiLimit,unitless,double,Resolving power max value on spectral axis,NO +em_resolution,Char.SpectralAxis.Resolution.Refval.value,m,double,Value of Resolution along the spectral axis,NO +em_stat_error,Char.SpectralAxis.Accuracy.StatError.Refval.value,m,double,Spectral coord statistical error,NO +o_ucd,Char.ObservableAxis.ucd,unitless,String,Nature of the observable axis,YES +o_unit,Char.ObservableAxis.unit,unitless, string,Units used for the observable values,NO +o_calib_status,Char.ObservableAxis.calibrationStatus,unitless,Enum string,Type of calibration for the observable coordinate,NO +o_stat_error,Char.ObservableAxis.Accuracy.StatError.Refval.value,units specified by o_unit,double,Statistical error on the Observable axis,NO +pol_xel,Char.PolarizationAxis.numBins,unitless,integer,Number of elements along the polarization axis,YES +pol_states,Char.PolarizationAxis.stateList,unitless,Enum string,List of polarization states present in the data file,NO +instrument_name,Provenance.ObsConfig.Instrument.name,unitless,string,The name of the instrument used for the observation,YES +proposal_id,Provenance.Proposal.identifier,string,string,Identifier of proposal to which observation belongs,NO From 45044162d770f1617e02e16592fce85fce900955 Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Mon, 16 May 2022 11:56:32 -0700 Subject: [PATCH 03/10] Apply ObsCore v1.1 erratum 1 --- yml/ivoa/ObsCore-v1.1-mandatory.csv | 4 ++-- yml/ivoa/ObsCore-v1.1-optional.csv | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/yml/ivoa/ObsCore-v1.1-mandatory.csv b/yml/ivoa/ObsCore-v1.1-mandatory.csv index 3f991c1b..3a6ca088 100644 --- a/yml/ivoa/ObsCore-v1.1-mandatory.csv +++ b/yml/ivoa/ObsCore-v1.1-mandatory.csv @@ -1,9 +1,9 @@ Column Name,Datatype,Size,Units,ObsCoreDM Utype,UCD,Principal,Index,Std -dataproduct_type,adql:VARCHAR,TBD,NULL,ObsDataset.dataProductType,meta.id,1,TBD,1 +dataproduct_type,adql:VARCHAR,TBD,NULL,ObsDataset.dataProductType,meta.code.class,1,TBD,1 calib_level,adql:INTEGER,NULL,NULL,ObsDataset.calibLevel,meta.code;obs.calib,1,TBD,1 obs_collection,adql:VARCHAR,TBD,NULL,DataID.collection,meta.id,1,TBD,1 obs_id,adql:VARCHAR,TBD,NULL,DataID.observationID,meta.id,1,TBD,1 -obs_publisher_did,adql:VARCHAR,TBD,NULL,Curation.publisherDID,meta.ref.uri;meta.curation,1,TBD,1 +obs_publisher_did,adql:VARCHAR,TBD,NULL,Curation.publisherDID,meta.ref.ivoid,1,TBD,1 access_url,adql:CLOB,NULL,NULL,Access.reference,meta.ref.url,1,0,1 access_format,adql:VARCHAR,NULL,NULL,Access.format,meta.code.mime,1,0,1 access_estsize,adql:BIGINT,NULL,kbyte,Access.size,phys.size;meta.file,1,0,1 diff --git a/yml/ivoa/ObsCore-v1.1-optional.csv b/yml/ivoa/ObsCore-v1.1-optional.csv index c0d7a358..ed7104a3 100644 --- a/yml/ivoa/ObsCore-v1.1-optional.csv +++ b/yml/ivoa/ObsCore-v1.1-optional.csv @@ -1,11 +1,11 @@ Column Name,Datatype,Size,Units,ObsCoreDM Utype,UCD,Principal,Index,Std -dataproduct_subtype,adql:VARCHAR,TBD,NULL,ObsDataset.dataProductSubtype,meta.id,1,TBD,1 +dataproduct_subtype,adql:VARCHAR,TBD,NULL,ObsDataset.dataProductSubtype,meta.code.class,1,TBD,1 target_class,adql:VARCHAR,TBD,NULL,Target.class,src.class,1,TBD,1 obs_creation_date,adql:TIMESTAMP,NULL,NULL,DataID.date,time;meta.dataset,1,TBD,1 obs_creator_name,adql:VARCHAR,TBD,NULL,DataID.creator,meta.id,1,TBD,1 obs_creator_did,adql:VARCHAR,TBD,NULL,DataID.creatorDID,meta.id,0,TBD,1 obs_title,adql:VARCHAR,200,NULL,DataID.title,meta.title;obs,1,0,1 -publisher_id,adql:VARCHAR,TBD,NULL,Curation.publisherID,meta.ref.uri;meta.curation,1,TBD,1 +publisher_id,adql:VARCHAR,TBD,NULL,Curation.publisherID,meta.ref.ivoid,1,TBD,1 bib_reference,adql:VARCHAR,TBD,NULL,Curation.reference,meta.bib,0,0,1 data_rights,adql:VARCHAR,NULL, NULL,Curation.rights,meta.code,0,0,1 obs_release_date,adql:TIMESTAMP,NULL,NULL,Curation.releaseDate,time.release,1,0,1 @@ -27,5 +27,5 @@ em_resolution,adql:DOUBLE,NULL,m,Char.SpectralAxis.Resolution.Refval.value,spect em_stat_error,adql:DOUBLE,NULL,m,Char.SpectralAxis.Accuracy.StatError.Refval.value,stat.error;em,0,0,1 o_unit,adql:VARCHAR,NULL,NULL,Char.ObservableAxis.unit,meta.unit,1,0,1 o_calib_status,adql:VARCHAR,NULL,NULL,Char.ObservableAxis.calibrationStatus,meta.code.qual,1,TBD,1 -o_stat_error,adql:DOUBLE,NULL,same units as the value of the o_unit attribute,Char.ObservableAxis.Accuracy.StatError.Refval.value,stat.error;phot.flux,0,0,1 +o_stat_error,adql:DOUBLE,NULL,same units as the value of the o_unit attribute,Char.ObservableAxis.Accuracy.StatError.Refval.value,stat.error,0,0,1 proposal_id,adql:VARCHAR,NULL,NULL,Provenance.Proposal.identifier,meta.id;obs.proposal,0,TBD,1 From 2085a5ab6ccc5fc8466d97fcb8a1977a7bce21b4 Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Mon, 16 May 2022 12:11:27 -0700 Subject: [PATCH 04/10] Apply ObsCore v1.1 erratum 2 --- yml/ivoa/ObsCore-v1.1-descriptions.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yml/ivoa/ObsCore-v1.1-descriptions.csv b/yml/ivoa/ObsCore-v1.1-descriptions.csv index 7ccf20c1..70b7d21c 100644 --- a/yml/ivoa/ObsCore-v1.1-descriptions.csv +++ b/yml/ivoa/ObsCore-v1.1-descriptions.csv @@ -21,7 +21,7 @@ access_estsize,Access.size,kbyte,int,Estimated size of dataset: in kilobytes,YES s_ra,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1,deg,double,"Central Spatial Position in ICRS; Right ascension",YES s_dec,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2,deg,double,"Central Spatial Position in ICRS; Declination",YES s_fov,Char.SpatialAxis.Coverage.Bounds.Extent.diameter,deg,double,Estimated size of the covered region as the diameter of a containing circle,YES -s_region,Char.SpatialAxis.Coverage.Support.Area,,string,Sky region covered by the data product (expressed in ICRS frame),YES +s_region,Char.SpatialAxis.Coverage.Support.Area,unitless,string,Sky region covered by the data product (expressed in ICRS frame),YES s_resolution,Char.SpatialAxis.Resolution.refval.value,arcsec,double,Spatial resolution of data as FWHM of PSF,YES s_xel1,Char.SpatialAxis.numBins1,unitless,integer,Number of elements along the first coordinate of the spatial axis,YES s_xel2,Char.SpatialAxis.numBins2,unitless,integer,Number of elements along the second coordinate of the spatial axis,YES @@ -33,7 +33,6 @@ s_calib_status,Char.SpatialAxis.calibrationStatus,unitless,Enum string,Type of c s_stat_error,Char.SpatialAxis.Accuracy.StatError.Refval.value,arcsec,double,Astrometric precision along the spatial axis,NO s_pixel_scale,Char.SpatialAxis.Sampling.RefVal.SamplingPeriod,arcsec,double,Sampling period in world coordinate units along the spatial axis,NO t_xel,Char.TimeAxis.numBins,unitless,integer,Number of elements along the time axis,YES -t_refpos,Char.TimeAxis.ReferencePosition,unitless,Enum string,"Time Axis Reference Position as defined in STC REC, Section 4.4.1.1.1",NO t_min,Char.TimeAxis.Coverage.Bounds.Limits.StartTime,d,double,Start time in MJD,YES t_max,Char.TimeAxis.Coverage.Bounds.Limits.StopTime,d,double,Stop time in MJD,YES t_exptime,Char.TimeAxis.Coverage.Support.Extent,s,double,Total exposure time,YES @@ -56,6 +55,7 @@ o_unit,Char.ObservableAxis.unit,unitless, string,Units used for the observable v o_calib_status,Char.ObservableAxis.calibrationStatus,unitless,Enum string,Type of calibration for the observable coordinate,NO o_stat_error,Char.ObservableAxis.Accuracy.StatError.Refval.value,units specified by o_unit,double,Statistical error on the Observable axis,NO pol_xel,Char.PolarizationAxis.numBins,unitless,integer,Number of elements along the polarization axis,YES -pol_states,Char.PolarizationAxis.stateList,unitless,Enum string,List of polarization states present in the data file,NO +pol_states,Char.PolarizationAxis.stateList,unitless,Enum string,List of polarization states present in the data file,YES +facility_name,Provenance.ObsConfig.Facility.name,unitless,string,"The name of the facility, telescope, or space craft used for the observation",YES instrument_name,Provenance.ObsConfig.Instrument.name,unitless,string,The name of the instrument used for the observation,YES -proposal_id,Provenance.Proposal.identifier,string,string,Identifier of proposal to which observation belongs,NO +proposal_id,Provenance.Proposal.identifier,unitless,string,Identifier of proposal to which observation belongs,NO From d8dd25b60f55d616c37aed4874318ea4d264065a Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Mon, 16 May 2022 12:17:58 -0700 Subject: [PATCH 05/10] Fix remaining issues with CSV export --- yml/ivoa/ObsCore-v1.1-descriptions.csv | 6 +++--- yml/ivoa/ObsCore-v1.1-mandatory.csv | 12 ++++++------ yml/ivoa/ObsCore-v1.1-optional.csv | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/yml/ivoa/ObsCore-v1.1-descriptions.csv b/yml/ivoa/ObsCore-v1.1-descriptions.csv index 70b7d21c..4ed02a53 100644 --- a/yml/ivoa/ObsCore-v1.1-descriptions.csv +++ b/yml/ivoa/ObsCore-v1.1-descriptions.csv @@ -22,7 +22,7 @@ s_ra,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1,deg,double,"C s_dec,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2,deg,double,"Central Spatial Position in ICRS; Declination",YES s_fov,Char.SpatialAxis.Coverage.Bounds.Extent.diameter,deg,double,Estimated size of the covered region as the diameter of a containing circle,YES s_region,Char.SpatialAxis.Coverage.Support.Area,unitless,string,Sky region covered by the data product (expressed in ICRS frame),YES -s_resolution,Char.SpatialAxis.Resolution.refval.value,arcsec,double,Spatial resolution of data as FWHM of PSF,YES +s_resolution,Char.SpatialAxis.Resolution.Refval.value,arcsec,double,Spatial resolution of data as FWHM of PSF,YES s_xel1,Char.SpatialAxis.numBins1,unitless,integer,Number of elements along the first coordinate of the spatial axis,YES s_xel2,Char.SpatialAxis.numBins2,unitless,integer,Number of elements along the second coordinate of the spatial axis,YES s_ucd,Char.SpatialAxis.ucd,unitless,string,"UCD for the nature of the spatial axis (pos or u,v data)",NO @@ -36,7 +36,7 @@ t_xel,Char.TimeAxis.numBins,unitless,integer,Number of elements along the time a t_min,Char.TimeAxis.Coverage.Bounds.Limits.StartTime,d,double,Start time in MJD,YES t_max,Char.TimeAxis.Coverage.Bounds.Limits.StopTime,d,double,Stop time in MJD,YES t_exptime,Char.TimeAxis.Coverage.Support.Extent,s,double,Total exposure time,YES -t_resolution,Char.TimeAxis.Resolution.Refval.valueResolution.Refval.value,s,double,Temporal resolution FWHM,YES +t_resolution,Char.TimeAxis.Resolution.Refval.value,s,double,Temporal resolution FWHM,YES t_calib_status,Char.TimeAxis.calibrationStatus,unitless,Enum string,Type of time coordinate calibration,NO t_stat_error,Char.TimeAxis.Accuracy.StatError.Refval.value,s,double,Time coord statistical error,NO em_xel,Char.SpectralAxis.numBins,unitless,integer,Number of elements along the spectral axis,YES @@ -51,7 +51,7 @@ em_res_power_max,Char.SpectralAxis.Resolution.ResolPower.HiLimit,unitless,double em_resolution,Char.SpectralAxis.Resolution.Refval.value,m,double,Value of Resolution along the spectral axis,NO em_stat_error,Char.SpectralAxis.Accuracy.StatError.Refval.value,m,double,Spectral coord statistical error,NO o_ucd,Char.ObservableAxis.ucd,unitless,String,Nature of the observable axis,YES -o_unit,Char.ObservableAxis.unit,unitless, string,Units used for the observable values,NO +o_unit,Char.ObservableAxis.unit,unitless,string,Units used for the observable values,NO o_calib_status,Char.ObservableAxis.calibrationStatus,unitless,Enum string,Type of calibration for the observable coordinate,NO o_stat_error,Char.ObservableAxis.Accuracy.StatError.Refval.value,units specified by o_unit,double,Statistical error on the Observable axis,NO pol_xel,Char.PolarizationAxis.numBins,unitless,integer,Number of elements along the polarization axis,YES diff --git a/yml/ivoa/ObsCore-v1.1-mandatory.csv b/yml/ivoa/ObsCore-v1.1-mandatory.csv index 3a6ca088..b813ed15 100644 --- a/yml/ivoa/ObsCore-v1.1-mandatory.csv +++ b/yml/ivoa/ObsCore-v1.1-mandatory.csv @@ -11,7 +11,7 @@ target_name,adql:VARCHAR,TBD,NULL,Target.name,meta.id;src,1,0,1 s_ra,adql:DOUBLE,NULL,deg,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1,pos.eq.ra,1,0,1 s_dec,adql:DOUBLE,NULL,deg,Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2,pos.eq.dec,1,0,1 s_fov,adql:DOUBLE,NULL,deg,Char.SpatialAxis.Coverage.Bounds.Extent.diameter,phys.angSize;instr.fov,1,0,1 -s_region,adql:REGION,NULL,,Char.SpatialAxis.Coverage.Support.Area,pos.outline;obs.field,1,0,1 +s_region,adql:REGION,NULL,NULL,Char.SpatialAxis.Coverage.Support.Area,pos.outline;obs.field,1,0,1 s_resolution,adql:DOUBLE,NULL,arcsec,Char.SpatialAxis.Resolution.Refval.value,pos.angResolution,1,TBD,1 s_xel1,adql:BIGINT,NULL,NULL,Char.SpatialAxis.numBins1,meta.number,1,TBD,1 s_xel2,adql:BIGINT,NULL,NULL,Char.SpatialAxis.numBins2,meta.number,1,TBD,1 @@ -19,13 +19,13 @@ t_min,adql:DOUBLE,NULL,d,Char.TimeAxis.Coverage.Bounds.Limits.StartTime,time.sta t_max,adql:DOUBLE,NULL,d,Char.TimeAxis.Coverage.Bounds.Limits.StopTime,time.end;obs.exposure,1,0,1 t_exptime,adql:DOUBLE,NULL,s,Char.TimeAxis.Coverage.Support.Extent,time.duration;obs.exposure,1,TBD,1 t_resolution,adql:DOUBLE,NULL,s,Char.TimeAxis.Resolution.Refval.value,time.resolution,1,0,1 -t_xel,adql: BIGINT,NULL,NULL,Char.TimeAxis.numBins,meta.number,1,TBD,1 +t_xel,adql:BIGINT,NULL,NULL,Char.TimeAxis.numBins,meta.number,1,TBD,1 em_min,adql:DOUBLE,NULL,m,Char.SpectralAxis.Coverage.Bounds.Limits.LoLimit,em.wl;stat.min,1,0,1 em_max,adql:DOUBLE,NULL,m,Char.SpectralAxis.Coverage.Bounds.Limits.HiLimit,em.wl;stat.max,1,0,1 em_res_power,adql:DOUBLE,NULL,NULL,Char.SpectralAxis.Resolution.ResolPower.refVal,spect.resolution,1,TBD,1 -em_xel,adql: BIGINT,NULL,NULL,Char.SpectralAxis.numBins,meta.number,1,TBD,1 +em_xel,adql:BIGINT,NULL,NULL,Char.SpectralAxis.numBins,meta.number,1,TBD,1 o_ucd,adql:VARCHAR,TBD,NULL,Char.ObservableAxis.ucd,meta.ucd,1,0,1 pol_states,adql:VARCHAR,TBD,NULL,Char.PolarizationAxis.stateList,meta.code;phys.polarization,1,0,1 -pol_xel,adql: BIGINT,NULL,NULL,Char.PolarizationAxis.numBins,meta.number,1,TBD,1 -facility_name,adql:VARCHAR,NULL,NULL,Provenance.ObsConfig.Facility.name,meta.id;instr.tel ,1,TBD,1 -Instrument_name,adql:VARCHAR,NULL ,NULL,Provenance.ObsConfig.Instrument.name,meta.id;instr,1,TBD,1 +pol_xel,adql:BIGINT,NULL,NULL,Char.PolarizationAxis.numBins,meta.number,1,TBD,1 +facility_name,adql:VARCHAR,NULL,NULL,Provenance.ObsConfig.Facility.name,meta.id;instr.tel,1,TBD,1 +instrument_name,adql:VARCHAR,NULL,NULL,Provenance.ObsConfig.Instrument.name,meta.id;instr,1,TBD,1 diff --git a/yml/ivoa/ObsCore-v1.1-optional.csv b/yml/ivoa/ObsCore-v1.1-optional.csv index ed7104a3..53e7b035 100644 --- a/yml/ivoa/ObsCore-v1.1-optional.csv +++ b/yml/ivoa/ObsCore-v1.1-optional.csv @@ -7,7 +7,7 @@ obs_creator_did,adql:VARCHAR,TBD,NULL,DataID.creatorDID,meta.id,0,TBD,1 obs_title,adql:VARCHAR,200,NULL,DataID.title,meta.title;obs,1,0,1 publisher_id,adql:VARCHAR,TBD,NULL,Curation.publisherID,meta.ref.ivoid,1,TBD,1 bib_reference,adql:VARCHAR,TBD,NULL,Curation.reference,meta.bib,0,0,1 -data_rights,adql:VARCHAR,NULL, NULL,Curation.rights,meta.code,0,0,1 +data_rights,adql:VARCHAR,NULL,NULL,Curation.rights,meta.code,0,0,1 obs_release_date,adql:TIMESTAMP,NULL,NULL,Curation.releaseDate,time.release,1,0,1 s_ucd,adql:VARCHAR,NULL,NULL,Char.SpatialAxis.ucd,meta.ucd,1,0,1 s_unit,adql:VARCHAR,NULL,NULL,Char.SpatialAxis.unit,meta.unit,1,0,1 @@ -21,7 +21,7 @@ t_stat_error,adql:DOUBLE,NULL,s,Char.TimeAxis.Accuracy.StatError.Refval.value,st em_ucd,adql:VARCHAR,NULL,NULL,Char.SpectralAxis.ucd,meta.ucd,1,TBD,1 em_unit,adql:VARCHAR,NULL,NULL,Char.SpectralAxis.unit,meta.unit,1,0,1 em_calib_status,adql:VARCHAR,NULL,NULL,Char.SpectralAxis.calibrationStatus,meta.code.qual,0,0,1 -em_res_power_min,adql:DOUBLE,NULL,NULL ,Char.SpectralAxis.Resolution.ResolPower.LoLimit,spect.resolution;stat.min,1,0,1 +em_res_power_min,adql:DOUBLE,NULL,NULL,Char.SpectralAxis.Resolution.ResolPower.LoLimit,spect.resolution;stat.min,1,0,1 em_res_power_max,adql:DOUBLE,NULL,NULL,Char.SpectralAxis.Resolution.ResolPower.HiLimit,spect.resolution;stat.max,1,0,1 em_resolution,adql:DOUBLE,NULL,m,Char.SpectralAxis.Resolution.Refval.value,spect.resolution;stat.mean,1,0,1 em_stat_error,adql:DOUBLE,NULL,m,Char.SpectralAxis.Accuracy.StatError.Refval.value,stat.error;em,0,0,1 From bab44dbf8f81b66ed1e48cdf164d8c892045a26b Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Thu, 26 May 2022 11:04:44 -0700 Subject: [PATCH 06/10] Initial version of ObsCore Felis YAML creator, and its output --- yml/dp02_obscore.yaml | 271 +++++++++++++++ yml/ivoa/make_obscore_nominal.py | 157 +++++++++ yml/ivoa/obscore_nominal.yaml | 550 +++++++++++++++++++++++++++++++ 3 files changed, 978 insertions(+) create mode 100644 yml/dp02_obscore.yaml create mode 100644 yml/ivoa/make_obscore_nominal.py create mode 100644 yml/ivoa/obscore_nominal.yaml diff --git a/yml/dp02_obscore.yaml b/yml/dp02_obscore.yaml new file mode 100644 index 00000000..f90e8a68 --- /dev/null +++ b/yml/dp02_obscore.yaml @@ -0,0 +1,271 @@ +--- +name: ivoa +"@id": "#ivoa_obscore" +description: ObsCore v1.1 attributes in ObsTAP realization +tables: +- name: ivoa.ObsCore + "@id": "#ObsCore" + description: Observation metadata in the ObsTAP relational realization of + the IVOA ObsCore data model + columns: + - name: dataproduct_type + "@id": "ObsCore.dataproduct_type" + description: Data product (file content) primary type + nullable: true + ivoa:ucd: meta.code.class + votable:utype: ObsDataset.dataProductType + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: dataproduct_subtype + "@id": "ObsCore.dataproduct_subtype" + description: Data product specific type + nullable: true + ivoa:ucd: meta.code.class + votable:utype: ObsDataset.dataProductSubtype + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: facility_name + "@id": "ObsCore.facility_name" + description: The name of the facility, telescope, or space craft used for the observation + nullable: true + ivoa:ucd: meta.id;instr.tel + votable:utype: Provenance.ObsConfig.Facility.name + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: calib_level + "@id": "ObsCore.calib_level" + description: "Calibration level of the observation: in {0, 1, 2, 3, 4}" + nullable: false + ivoa:ucd: meta.code;obs.calib + votable:utype: ObsDataset.calibLevel + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: target_name + "@id": "ObsCore.target_name" + description: Object of interest + nullable: true + ivoa:ucd: meta.id;src + votable:utype: Target.name + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_id + "@id": "ObsCore.obs_id" + description: Internal ID given by the ObsTAP service + nullable: false + ivoa:ucd: meta.id + votable:utype: DataID.observationID + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_collection + "@id": "ObsCore.obs_collection" + description: Name of the data collection + nullable: false + ivoa:ucd: meta.id + votable:utype: DataID.collection + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_publisher_did + "@id": "ObsCore.obs_publisher_did" + description: ID for the Dataset given by the publisher + nullable: false + ivoa:ucd: meta.ref.ivoid + votable:utype: Curation.publisherDID + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: access_url + "@id": "ObsCore.access_url" + description: URL used to access dataset + nullable: true + ivoa:ucd: meta.ref.url + votable:utype: Access.reference + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: access_format + "@id": "ObsCore.access_format" + description: Content format of the dataset + nullable: true + ivoa:ucd: meta.code.mime + votable:utype: Access.format + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_ra + "@id": "ObsCore.s_ra" + description: Central Spatial Position in ICRS; Right ascension + nullable: true + ivoa:ucd: pos.eq.ra + votable:utype: Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1 + tap:std: 1 + tap:principal: 1 + ivoa:unit: deg + - name: s_dec + "@id": "ObsCore.s_dec" + description: Central Spatial Position in ICRS; Declination + nullable: true + ivoa:ucd: pos.eq.dec + votable:utype: Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2 + tap:std: 1 + tap:principal: 1 + ivoa:unit: deg + - name: s_fov + "@id": "ObsCore.s_fov" + description: Estimated size of the covered region as the diameter of a containing circle + nullable: true + ivoa:ucd: phys.angSize;instr.fov + votable:utype: Char.SpatialAxis.Coverage.Bounds.Extent.diameter + tap:std: 1 + tap:principal: 1 + ivoa:unit: deg + - name: s_region + "@id": "ObsCore.s_region" + description: Sky region covered by the data product (expressed in ICRS frame) + nullable: true + ivoa:ucd: pos.outline;obs.field + votable:utype: Char.SpatialAxis.Coverage.Support.Area + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_resolution + "@id": "ObsCore.s_resolution" + description: Spatial resolution of data as FWHM of PSF + nullable: true + ivoa:ucd: pos.angResolution + votable:utype: Char.SpatialAxis.Resolution.Refval.value + tap:std: 1 + tap:principal: 1 + ivoa:unit: arcsec + - name: s_xel1 + "@id": "ObsCore.s_xel1" + description: Number of elements along the first coordinate of the spatial axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.SpatialAxis.numBins1 + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_xel2 + "@id": "ObsCore.s_xel2" + description: Number of elements along the second coordinate of the spatial axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.SpatialAxis.numBins2 + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: t_xel + "@id": "ObsCore.t_xel" + description: Number of elements along the time axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.TimeAxis.numBins + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: t_min + "@id": "ObsCore.t_min" + description: Start time in MJD + nullable: true + ivoa:ucd: time.start;obs.exposure + votable:utype: Char.TimeAxis.Coverage.Bounds.Limits.StartTime + tap:std: 1 + tap:principal: 1 + ivoa:unit: d + - name: t_max + "@id": "ObsCore.t_max" + description: Stop time in MJD + nullable: true + ivoa:ucd: time.end;obs.exposure + votable:utype: Char.TimeAxis.Coverage.Bounds.Limits.StopTime + tap:std: 1 + tap:principal: 1 + ivoa:unit: d + - name: t_exptime + "@id": "ObsCore.t_exptime" + description: Total exposure time + nullable: true + ivoa:ucd: time.duration;obs.exposure + votable:utype: Char.TimeAxis.Coverage.Support.Extent + tap:std: 1 + tap:principal: 1 + ivoa:unit: s + - name: t_resolution + "@id": "ObsCore.t_resolution" + description: Temporal resolution FWHM + nullable: true + ivoa:ucd: time.resolution + votable:utype: Char.TimeAxis.Resolution.Refval.value + tap:std: 1 + tap:principal: 1 + ivoa:unit: s + - name: em_xel + "@id": "ObsCore.em_xel" + description: Number of elements along the spectral axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.SpectralAxis.numBins + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: em_min + "@id": "ObsCore.em_min" + description: start in spectral coordinates + nullable: true + ivoa:ucd: em.wl;stat.min + votable:utype: Char.SpectralAxis.Coverage.Bounds.Limits.LoLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: m + - name: em_max + "@id": "ObsCore.em_max" + description: stop in spectral coordinates + nullable: true + ivoa:ucd: em.wl;stat.max + votable:utype: Char.SpectralAxis.Coverage.Bounds.Limits.HiLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: m + - name: em_res_power + "@id": "ObsCore.em_res_power" + description: Value of the resolving power along the spectral axis (R) + nullable: true + ivoa:ucd: spect.resolution + votable:utype: Char.SpectralAxis.Resolution.ResolPower.refVal + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: o_ucd + "@id": "ObsCore.o_ucd" + description: Nature of the observable axis + nullable: true + ivoa:ucd: meta.ucd + votable:utype: Char.ObservableAxis.ucd + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: pol_xel + "@id": "ObsCore.pol_xel" + description: Number of elements along the polarization axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.PolarizationAxis.numBins + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: instrument_name + "@id": "ObsCore.instrument_name" + description: The name of the instrument used for the observation + nullable: true + ivoa:ucd: meta.id;instr + votable:utype: Provenance.ObsConfig.Instrument.name + tap:std: 1 + tap:principal: 1 + ivoa:unit: diff --git a/yml/ivoa/make_obscore_nominal.py b/yml/ivoa/make_obscore_nominal.py new file mode 100644 index 00000000..fad5f051 --- /dev/null +++ b/yml/ivoa/make_obscore_nominal.py @@ -0,0 +1,157 @@ +# This file is part of sdm_schemas/yml/ivoa. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Generator for Felis for the nominal ObsCore data model + +This stand-alone program takes tabular data on the columns in the +ObsTAP realization of the ObsCore data model, taken directly from +the IVOA ObsCore standard, with errata applied, and converted to +CSV, and produces a Rubin-flavored "Felis" description in YAML of +the data model. The result is intended to be used as input to the +creation of operation Felis descriptions of ObsCore-formatted +tables on RSP services. + +Output is produced to stdout and is intended to be redirected to a file. +""" + +import argparse +import csv + +def to_yaml( text: str ) -> str: + """Rough conversion of a string to an acceptable format for YAML. + + Currently only handles issues actually arising in the ObsCore table descriptions. + As of this writing only the colon-followed-by-space rule is considered, and + in that case the string is simply wrapped in double quotes. + + Parameters + ---------- + text : `str` + String to be converted. + + Returns + ------- + yamltext : `str` + String in an acceptable YAML format. + """ + if ": " in text: + return "\"" + text + "\"" + else: + return text + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Produce Felis YAML for ObsCore attributes in the ObsTAP realization", + epilog="If no attribute list is provided, creates YAML for the entire" + " set of defined ObsCore attributes in the standard," + " with optional attributes noted." ) + parser.add_argument('attr_file', nargs='?', default='', + help="File with list of attributes to be used, one name per line (default: all)") + args = parser.parse_args() + + # Note the four key ObsCore attributes that must be NOT NULL (in v1.1). + not_null = ['calib_level', 'obs_collection', 'obs_id', 'obs_publisher_did'] + + # First, process Table 5 from the standard, which is primarily valuable + # for containing text descriptions of each column in the model. + desc_data = dict() + with open('ObsCore-v1.1-descriptions.csv', newline='') as desc_file: + desc_reader = csv.DictReader(desc_file) + for r in desc_reader: + if r['Column Name'] in desc_data: + raise RuntimeError( "Duplicate record in descriptions: " + repr(r) ) + desc_data[r['Column Name']] = r + + # Second, process Tables 6 and 7, which provide the more reliable detailed + # specifications of type, UCD, Utype, and so on. + col_data = dict() + with open('ObsCore-v1.1-mandatory.csv', newline='') as col_file: + col_reader = csv.DictReader(col_file) + for r in col_reader: + if r['Column Name'] in col_data: + raise RuntimeError( "Duplicate record in definitions: " + repr(r) ) + r['optional'] = False # Table 6 attributes are mandatory + col_data[r['Column Name']] = r + + with open('ObsCore-v1.1-optional.csv', newline='') as col_file: + col_reader = csv.DictReader(col_file) + for r in col_reader: + if r['Column Name'] in col_data: + raise RuntimeError( "Duplicate record in optional definitions: " + repr(r) ) + r['optional'] = True # Table 7 attributes are optional + col_data[r['Column Name']] = r + + # Process arguments to determine which attributes to emit. + # Default: output all attributes and mark optional ones. + col_list = col_data.keys() + mark_opt = True + + # If a file was supplied, take the first token in each row as an attribute name. + if args.attr_file != '': + with open(args.attr_file, newline='') as attr_file: + mark_opt = False + attr_reader = csv.reader(attr_file) + col_list = [ attr_row[0] for attr_row in attr_reader ] + if len(col_list) < 1: + raise RuntimeError( "No attributes found in file '" + args.attr_file + "'" ) + + #TODO - validate the list to ensure it contains only known attributes, + # and all the mandatory ones. + + # emit file-level header describing the file and the single table it defines + print("---\n" + "name: ivoa\n" + "\"@id\": \"#ivoa_obscore\"\n" + "description: ObsCore v1.1 attributes in ObsTAP realization\n" + "tables:\n" + "- name: ivoa.ObsCore\n" + " \"@id\": \"#ObsCore\"\n" + " description: Observation metadata in the ObsTAP relational realization of\n" + " the IVOA ObsCore data model\n" + " columns:" + ) + + # Loop over columns in the data model and emit the paragraphs for each one. + # The keys for the various properties are a bit arbitrary as to whether they + # have "ivoa:", "tap:", etc. prefixes. This is what is currently required + # in Felis processing. + for col in col_list: + print( + " - name: " + col + + (" # (optional)" if mark_opt and col_data[col]['optional'] else "") + "\n" + " \"@id\": \"ObsCore." + col + "\"\n" + " description: " + to_yaml(desc_data[col]['Description']) + "\n" + " nullable: " + ( "false" if col in not_null else "true" ) + "\n" + " ivoa:ucd: " + col_data[col]['UCD'] + "\n" + " votable:utype: " + col_data[col]['ObsCoreDM Utype'] + "\n" + " tap:std: " + col_data[col]['Std'] + "\n" + " tap:principal: " + col_data[col]['Principal'] + ) + + # In the ObsCore document, from which the CSV files were copied, columns with + # suggested null values have the string "NULL", which we should not emit into Felis. + col_units = col_data[col]['Units'] + if col_units == 'NULL': col_units = '' + print( + " ivoa:unit: " + col_units + ) + + diff --git a/yml/ivoa/obscore_nominal.yaml b/yml/ivoa/obscore_nominal.yaml new file mode 100644 index 00000000..f0040eb1 --- /dev/null +++ b/yml/ivoa/obscore_nominal.yaml @@ -0,0 +1,550 @@ +--- +name: ivoa +"@id": "#ivoa_obscore" +description: ObsCore v1.1 attributes in ObsTAP realization +tables: +- name: ivoa.ObsCore + "@id": "#ObsCore" + description: Observation metadata in the ObsTAP relational realization of + the IVOA ObsCore data model + columns: + - name: dataproduct_type + "@id": "ObsCore.dataproduct_type" + description: Data product (file content) primary type + nullable: true + ivoa:ucd: meta.code.class + votable:utype: ObsDataset.dataProductType + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: calib_level + "@id": "ObsCore.calib_level" + description: "Calibration level of the observation: in {0, 1, 2, 3, 4}" + nullable: false + ivoa:ucd: meta.code;obs.calib + votable:utype: ObsDataset.calibLevel + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_collection + "@id": "ObsCore.obs_collection" + description: Name of the data collection + nullable: false + ivoa:ucd: meta.id + votable:utype: DataID.collection + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_id + "@id": "ObsCore.obs_id" + description: Internal ID given by the ObsTAP service + nullable: false + ivoa:ucd: meta.id + votable:utype: DataID.observationID + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_publisher_did + "@id": "ObsCore.obs_publisher_did" + description: ID for the Dataset given by the publisher + nullable: false + ivoa:ucd: meta.ref.ivoid + votable:utype: Curation.publisherDID + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: access_url + "@id": "ObsCore.access_url" + description: URL used to access dataset + nullable: true + ivoa:ucd: meta.ref.url + votable:utype: Access.reference + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: access_format + "@id": "ObsCore.access_format" + description: Content format of the dataset + nullable: true + ivoa:ucd: meta.code.mime + votable:utype: Access.format + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: access_estsize + "@id": "ObsCore.access_estsize" + description: "Estimated size of dataset: in kilobytes" + nullable: true + ivoa:ucd: phys.size;meta.file + votable:utype: Access.size + tap:std: 1 + tap:principal: 1 + ivoa:unit: kbyte + - name: target_name + "@id": "ObsCore.target_name" + description: Object of interest + nullable: true + ivoa:ucd: meta.id;src + votable:utype: Target.name + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_ra + "@id": "ObsCore.s_ra" + description: Central Spatial Position in ICRS; Right ascension + nullable: true + ivoa:ucd: pos.eq.ra + votable:utype: Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1 + tap:std: 1 + tap:principal: 1 + ivoa:unit: deg + - name: s_dec + "@id": "ObsCore.s_dec" + description: Central Spatial Position in ICRS; Declination + nullable: true + ivoa:ucd: pos.eq.dec + votable:utype: Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2 + tap:std: 1 + tap:principal: 1 + ivoa:unit: deg + - name: s_fov + "@id": "ObsCore.s_fov" + description: Estimated size of the covered region as the diameter of a containing circle + nullable: true + ivoa:ucd: phys.angSize;instr.fov + votable:utype: Char.SpatialAxis.Coverage.Bounds.Extent.diameter + tap:std: 1 + tap:principal: 1 + ivoa:unit: deg + - name: s_region + "@id": "ObsCore.s_region" + description: Sky region covered by the data product (expressed in ICRS frame) + nullable: true + ivoa:ucd: pos.outline;obs.field + votable:utype: Char.SpatialAxis.Coverage.Support.Area + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_resolution + "@id": "ObsCore.s_resolution" + description: Spatial resolution of data as FWHM of PSF + nullable: true + ivoa:ucd: pos.angResolution + votable:utype: Char.SpatialAxis.Resolution.Refval.value + tap:std: 1 + tap:principal: 1 + ivoa:unit: arcsec + - name: s_xel1 + "@id": "ObsCore.s_xel1" + description: Number of elements along the first coordinate of the spatial axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.SpatialAxis.numBins1 + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_xel2 + "@id": "ObsCore.s_xel2" + description: Number of elements along the second coordinate of the spatial axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.SpatialAxis.numBins2 + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: t_min + "@id": "ObsCore.t_min" + description: Start time in MJD + nullable: true + ivoa:ucd: time.start;obs.exposure + votable:utype: Char.TimeAxis.Coverage.Bounds.Limits.StartTime + tap:std: 1 + tap:principal: 1 + ivoa:unit: d + - name: t_max + "@id": "ObsCore.t_max" + description: Stop time in MJD + nullable: true + ivoa:ucd: time.end;obs.exposure + votable:utype: Char.TimeAxis.Coverage.Bounds.Limits.StopTime + tap:std: 1 + tap:principal: 1 + ivoa:unit: d + - name: t_exptime + "@id": "ObsCore.t_exptime" + description: Total exposure time + nullable: true + ivoa:ucd: time.duration;obs.exposure + votable:utype: Char.TimeAxis.Coverage.Support.Extent + tap:std: 1 + tap:principal: 1 + ivoa:unit: s + - name: t_resolution + "@id": "ObsCore.t_resolution" + description: Temporal resolution FWHM + nullable: true + ivoa:ucd: time.resolution + votable:utype: Char.TimeAxis.Resolution.Refval.value + tap:std: 1 + tap:principal: 1 + ivoa:unit: s + - name: t_xel + "@id": "ObsCore.t_xel" + description: Number of elements along the time axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.TimeAxis.numBins + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: em_min + "@id": "ObsCore.em_min" + description: start in spectral coordinates + nullable: true + ivoa:ucd: em.wl;stat.min + votable:utype: Char.SpectralAxis.Coverage.Bounds.Limits.LoLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: m + - name: em_max + "@id": "ObsCore.em_max" + description: stop in spectral coordinates + nullable: true + ivoa:ucd: em.wl;stat.max + votable:utype: Char.SpectralAxis.Coverage.Bounds.Limits.HiLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: m + - name: em_res_power + "@id": "ObsCore.em_res_power" + description: Value of the resolving power along the spectral axis (R) + nullable: true + ivoa:ucd: spect.resolution + votable:utype: Char.SpectralAxis.Resolution.ResolPower.refVal + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: em_xel + "@id": "ObsCore.em_xel" + description: Number of elements along the spectral axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.SpectralAxis.numBins + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: o_ucd + "@id": "ObsCore.o_ucd" + description: Nature of the observable axis + nullable: true + ivoa:ucd: meta.ucd + votable:utype: Char.ObservableAxis.ucd + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: pol_states + "@id": "ObsCore.pol_states" + description: List of polarization states present in the data file + nullable: true + ivoa:ucd: meta.code;phys.polarization + votable:utype: Char.PolarizationAxis.stateList + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: pol_xel + "@id": "ObsCore.pol_xel" + description: Number of elements along the polarization axis + nullable: true + ivoa:ucd: meta.number + votable:utype: Char.PolarizationAxis.numBins + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: facility_name + "@id": "ObsCore.facility_name" + description: The name of the facility, telescope, or space craft used for the observation + nullable: true + ivoa:ucd: meta.id;instr.tel + votable:utype: Provenance.ObsConfig.Facility.name + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: instrument_name + "@id": "ObsCore.instrument_name" + description: The name of the instrument used for the observation + nullable: true + ivoa:ucd: meta.id;instr + votable:utype: Provenance.ObsConfig.Instrument.name + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: dataproduct_subtype # (optional) + "@id": "ObsCore.dataproduct_subtype" + description: Data product specific type + nullable: true + ivoa:ucd: meta.code.class + votable:utype: ObsDataset.dataProductSubtype + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: target_class # (optional) + "@id": "ObsCore.target_class" + description: Class of the Target object as in SSA + nullable: true + ivoa:ucd: src.class + votable:utype: Target.class + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_creation_date # (optional) + "@id": "ObsCore.obs_creation_date" + description: Date when the dataset was created + nullable: true + ivoa:ucd: time;meta.dataset + votable:utype: DataID.date + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_creator_name # (optional) + "@id": "ObsCore.obs_creator_name" + description: Name of the creator of the data + nullable: true + ivoa:ucd: meta.id + votable:utype: DataID.creator + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: obs_creator_did # (optional) + "@id": "ObsCore.obs_creator_did" + description: IVOA dataset identifier given by the creator + nullable: true + ivoa:ucd: meta.id + votable:utype: DataID.creatorDID + tap:std: 1 + tap:principal: 0 + ivoa:unit: + - name: obs_title # (optional) + "@id": "ObsCore.obs_title" + description: Brief description of dataset in free format + nullable: true + ivoa:ucd: meta.title;obs + votable:utype: DataID.title + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: publisher_id # (optional) + "@id": "ObsCore.publisher_id" + description: IVOA-ID for the Publisher + nullable: true + ivoa:ucd: meta.ref.ivoid + votable:utype: Curation.publisherID + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: bib_reference # (optional) + "@id": "ObsCore.bib_reference" + description: Service bibliographic reference + nullable: true + ivoa:ucd: meta.bib + votable:utype: Curation.reference + tap:std: 1 + tap:principal: 0 + ivoa:unit: + - name: data_rights # (optional) + "@id": "ObsCore.data_rights" + description: Public/Secure/Proprietary/ + nullable: true + ivoa:ucd: meta.code + votable:utype: Curation.rights + tap:std: 1 + tap:principal: 0 + ivoa:unit: + - name: obs_release_date # (optional) + "@id": "ObsCore.obs_release_date" + description: Observation release date (ISO 8601) + nullable: true + ivoa:ucd: time.release + votable:utype: Curation.releaseDate + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_ucd # (optional) + "@id": "ObsCore.s_ucd" + description: UCD for the nature of the spatial axis (pos or u,v data) + nullable: true + ivoa:ucd: meta.ucd + votable:utype: Char.SpatialAxis.ucd + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_unit # (optional) + "@id": "ObsCore.s_unit" + description: Unit used for spatial axis + nullable: true + ivoa:ucd: meta.unit + votable:utype: Char.SpatialAxis.unit + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_resolution_min # (optional) + "@id": "ObsCore.s_resolution_min" + description: Resolution min value on spatial axis (FHWM of PSF) + nullable: true + ivoa:ucd: pos.angResolution;stat.min + votable:utype: Char.SpatialAxis.Resolution.Bounds.Limits.LoLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: arcsec + - name: s_resolution_max # (optional) + "@id": "ObsCore.s_resolution_max" + description: Resolution max value on spatial axis + nullable: true + ivoa:ucd: pos.angResolution;stat.max + votable:utype: Char.SpatialAxis.Resolution.Bounds.Limits.HiLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: arcsec + - name: s_pixel_scale # (optional) + "@id": "ObsCore.s_pixel_scale" + description: Sampling period in world coordinate units along the spatial axis + nullable: true + ivoa:ucd: phys.angSize;instr.pixel + votable:utype: Char.SpatialAxis.Sampling.RefVal.SamplingPeriod + tap:std: 1 + tap:principal: 1 + ivoa:unit: arcsec + - name: s_calib_status # (optional) + "@id": "ObsCore.s_calib_status" + description: Type of calibration along the spatial axis + nullable: true + ivoa:ucd: meta.code.qual + votable:utype: Char.SpatialAxis.calibrationStatus + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: s_stat_error # (optional) + "@id": "ObsCore.s_stat_error" + description: Astrometric precision along the spatial axis + nullable: true + ivoa:ucd: stat.error;pos.eq + votable:utype: Char.SpatialAxis.Accuracy.StatError.Refval.value + tap:std: 1 + tap:principal: 0 + ivoa:unit: arcsec + - name: t_calib_status # (optional) + "@id": "ObsCore.t_calib_status" + description: Type of time coordinate calibration + nullable: true + ivoa:ucd: meta.code.qual + votable:utype: Char.TimeAxis.calibrationStatus + tap:std: 1 + tap:principal: 0 + ivoa:unit: + - name: t_stat_error # (optional) + "@id": "ObsCore.t_stat_error" + description: Time coord statistical error + nullable: true + ivoa:ucd: stat.error;time + votable:utype: Char.TimeAxis.Accuracy.StatError.Refval.value + tap:std: 1 + tap:principal: 0 + ivoa:unit: s + - name: em_ucd # (optional) + "@id": "ObsCore.em_ucd" + description: Nature of the spectral axis + nullable: true + ivoa:ucd: meta.ucd + votable:utype: Char.SpectralAxis.ucd + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: em_unit # (optional) + "@id": "ObsCore.em_unit" + description: Units along the spectral axis + nullable: true + ivoa:ucd: meta.unit + votable:utype: Char.SpectralAxis.unit + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: em_calib_status # (optional) + "@id": "ObsCore.em_calib_status" + description: Type of spectral coord calibration + nullable: true + ivoa:ucd: meta.code.qual + votable:utype: Char.SpectralAxis.calibrationStatus + tap:std: 1 + tap:principal: 0 + ivoa:unit: + - name: em_res_power_min # (optional) + "@id": "ObsCore.em_res_power_min" + description: Resolving power min value on spectral axis + nullable: true + ivoa:ucd: spect.resolution;stat.min + votable:utype: Char.SpectralAxis.Resolution.ResolPower.LoLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: em_res_power_max # (optional) + "@id": "ObsCore.em_res_power_max" + description: Resolving power max value on spectral axis + nullable: true + ivoa:ucd: spect.resolution;stat.max + votable:utype: Char.SpectralAxis.Resolution.ResolPower.HiLimit + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: em_resolution # (optional) + "@id": "ObsCore.em_resolution" + description: Value of Resolution along the spectral axis + nullable: true + ivoa:ucd: spect.resolution;stat.mean + votable:utype: Char.SpectralAxis.Resolution.Refval.value + tap:std: 1 + tap:principal: 1 + ivoa:unit: m + - name: em_stat_error # (optional) + "@id": "ObsCore.em_stat_error" + description: Spectral coord statistical error + nullable: true + ivoa:ucd: stat.error;em + votable:utype: Char.SpectralAxis.Accuracy.StatError.Refval.value + tap:std: 1 + tap:principal: 0 + ivoa:unit: m + - name: o_unit # (optional) + "@id": "ObsCore.o_unit" + description: Units used for the observable values + nullable: true + ivoa:ucd: meta.unit + votable:utype: Char.ObservableAxis.unit + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: o_calib_status # (optional) + "@id": "ObsCore.o_calib_status" + description: Type of calibration for the observable coordinate + nullable: true + ivoa:ucd: meta.code.qual + votable:utype: Char.ObservableAxis.calibrationStatus + tap:std: 1 + tap:principal: 1 + ivoa:unit: + - name: o_stat_error # (optional) + "@id": "ObsCore.o_stat_error" + description: Statistical error on the Observable axis + nullable: true + ivoa:ucd: stat.error + votable:utype: Char.ObservableAxis.Accuracy.StatError.Refval.value + tap:std: 1 + tap:principal: 0 + ivoa:unit: same units as the value of the o_unit attribute + - name: proposal_id # (optional) + "@id": "ObsCore.proposal_id" + description: Identifier of proposal to which observation belongs + nullable: true + ivoa:ucd: meta.id;obs.proposal + votable:utype: Provenance.Proposal.identifier + tap:std: 1 + tap:principal: 0 + ivoa:unit: From 2264b927dee64bd26917c10527cad21817966963 Mon Sep 17 00:00:00 2001 From: Fritz Mueller Date: Fri, 27 May 2022 12:56:47 -0700 Subject: [PATCH 07/10] linter fixes --- yml/dp02_obscore.yaml | 38 ++++----- yml/ivoa/obscore_nominal.yaml | 142 +++++++++++++++++----------------- 2 files changed, 90 insertions(+), 90 deletions(-) diff --git a/yml/dp02_obscore.yaml b/yml/dp02_obscore.yaml index f90e8a68..296f4423 100644 --- a/yml/dp02_obscore.yaml +++ b/yml/dp02_obscore.yaml @@ -16,7 +16,7 @@ tables: votable:utype: ObsDataset.dataProductType tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: dataproduct_subtype "@id": "ObsCore.dataproduct_subtype" description: Data product specific type @@ -25,7 +25,7 @@ tables: votable:utype: ObsDataset.dataProductSubtype tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: facility_name "@id": "ObsCore.facility_name" description: The name of the facility, telescope, or space craft used for the observation @@ -34,7 +34,7 @@ tables: votable:utype: Provenance.ObsConfig.Facility.name tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: calib_level "@id": "ObsCore.calib_level" description: "Calibration level of the observation: in {0, 1, 2, 3, 4}" @@ -43,7 +43,7 @@ tables: votable:utype: ObsDataset.calibLevel tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: target_name "@id": "ObsCore.target_name" description: Object of interest @@ -52,7 +52,7 @@ tables: votable:utype: Target.name tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: obs_id "@id": "ObsCore.obs_id" description: Internal ID given by the ObsTAP service @@ -61,7 +61,7 @@ tables: votable:utype: DataID.observationID tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: obs_collection "@id": "ObsCore.obs_collection" description: Name of the data collection @@ -70,7 +70,7 @@ tables: votable:utype: DataID.collection tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: obs_publisher_did "@id": "ObsCore.obs_publisher_did" description: ID for the Dataset given by the publisher @@ -79,7 +79,7 @@ tables: votable:utype: Curation.publisherDID tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: access_url "@id": "ObsCore.access_url" description: URL used to access dataset @@ -88,7 +88,7 @@ tables: votable:utype: Access.reference tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: access_format "@id": "ObsCore.access_format" description: Content format of the dataset @@ -97,7 +97,7 @@ tables: votable:utype: Access.format tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: s_ra "@id": "ObsCore.s_ra" description: Central Spatial Position in ICRS; Right ascension @@ -133,7 +133,7 @@ tables: votable:utype: Char.SpatialAxis.Coverage.Support.Area tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: s_resolution "@id": "ObsCore.s_resolution" description: Spatial resolution of data as FWHM of PSF @@ -151,7 +151,7 @@ tables: votable:utype: Char.SpatialAxis.numBins1 tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: s_xel2 "@id": "ObsCore.s_xel2" description: Number of elements along the second coordinate of the spatial axis @@ -160,7 +160,7 @@ tables: votable:utype: Char.SpatialAxis.numBins2 tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: t_xel "@id": "ObsCore.t_xel" description: Number of elements along the time axis @@ -169,7 +169,7 @@ tables: votable:utype: Char.TimeAxis.numBins tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: t_min "@id": "ObsCore.t_min" description: Start time in MJD @@ -214,7 +214,7 @@ tables: votable:utype: Char.SpectralAxis.numBins tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: em_min "@id": "ObsCore.em_min" description: start in spectral coordinates @@ -241,7 +241,7 @@ tables: votable:utype: Char.SpectralAxis.Resolution.ResolPower.refVal tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: o_ucd "@id": "ObsCore.o_ucd" description: Nature of the observable axis @@ -250,7 +250,7 @@ tables: votable:utype: Char.ObservableAxis.ucd tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: pol_xel "@id": "ObsCore.pol_xel" description: Number of elements along the polarization axis @@ -259,7 +259,7 @@ tables: votable:utype: Char.PolarizationAxis.numBins tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: instrument_name "@id": "ObsCore.instrument_name" description: The name of the instrument used for the observation @@ -268,4 +268,4 @@ tables: votable:utype: Provenance.ObsConfig.Instrument.name tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: diff --git a/yml/ivoa/obscore_nominal.yaml b/yml/ivoa/obscore_nominal.yaml index f0040eb1..6347a543 100644 --- a/yml/ivoa/obscore_nominal.yaml +++ b/yml/ivoa/obscore_nominal.yaml @@ -16,7 +16,7 @@ tables: votable:utype: ObsDataset.dataProductType tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: calib_level "@id": "ObsCore.calib_level" description: "Calibration level of the observation: in {0, 1, 2, 3, 4}" @@ -25,7 +25,7 @@ tables: votable:utype: ObsDataset.calibLevel tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: obs_collection "@id": "ObsCore.obs_collection" description: Name of the data collection @@ -34,7 +34,7 @@ tables: votable:utype: DataID.collection tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: obs_id "@id": "ObsCore.obs_id" description: Internal ID given by the ObsTAP service @@ -43,7 +43,7 @@ tables: votable:utype: DataID.observationID tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: obs_publisher_did "@id": "ObsCore.obs_publisher_did" description: ID for the Dataset given by the publisher @@ -52,7 +52,7 @@ tables: votable:utype: Curation.publisherDID tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: access_url "@id": "ObsCore.access_url" description: URL used to access dataset @@ -61,7 +61,7 @@ tables: votable:utype: Access.reference tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: access_format "@id": "ObsCore.access_format" description: Content format of the dataset @@ -70,7 +70,7 @@ tables: votable:utype: Access.format tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: access_estsize "@id": "ObsCore.access_estsize" description: "Estimated size of dataset: in kilobytes" @@ -88,7 +88,7 @@ tables: votable:utype: Target.name tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: s_ra "@id": "ObsCore.s_ra" description: Central Spatial Position in ICRS; Right ascension @@ -124,7 +124,7 @@ tables: votable:utype: Char.SpatialAxis.Coverage.Support.Area tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: s_resolution "@id": "ObsCore.s_resolution" description: Spatial resolution of data as FWHM of PSF @@ -142,7 +142,7 @@ tables: votable:utype: Char.SpatialAxis.numBins1 tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: s_xel2 "@id": "ObsCore.s_xel2" description: Number of elements along the second coordinate of the spatial axis @@ -151,7 +151,7 @@ tables: votable:utype: Char.SpatialAxis.numBins2 tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: t_min "@id": "ObsCore.t_min" description: Start time in MJD @@ -196,7 +196,7 @@ tables: votable:utype: Char.TimeAxis.numBins tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: em_min "@id": "ObsCore.em_min" description: start in spectral coordinates @@ -223,7 +223,7 @@ tables: votable:utype: Char.SpectralAxis.Resolution.ResolPower.refVal tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: em_xel "@id": "ObsCore.em_xel" description: Number of elements along the spectral axis @@ -232,7 +232,7 @@ tables: votable:utype: Char.SpectralAxis.numBins tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: o_ucd "@id": "ObsCore.o_ucd" description: Nature of the observable axis @@ -241,7 +241,7 @@ tables: votable:utype: Char.ObservableAxis.ucd tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: pol_states "@id": "ObsCore.pol_states" description: List of polarization states present in the data file @@ -250,7 +250,7 @@ tables: votable:utype: Char.PolarizationAxis.stateList tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: pol_xel "@id": "ObsCore.pol_xel" description: Number of elements along the polarization axis @@ -259,7 +259,7 @@ tables: votable:utype: Char.PolarizationAxis.numBins tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: facility_name "@id": "ObsCore.facility_name" description: The name of the facility, telescope, or space craft used for the observation @@ -268,7 +268,7 @@ tables: votable:utype: Provenance.ObsConfig.Facility.name tap:std: 1 tap:principal: 1 - ivoa:unit: + ivoa:unit: - name: instrument_name "@id": "ObsCore.instrument_name" description: The name of the instrument used for the observation @@ -277,8 +277,8 @@ tables: votable:utype: Provenance.ObsConfig.Instrument.name tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: dataproduct_subtype # (optional) + ivoa:unit: + - name: dataproduct_subtype # (optional) "@id": "ObsCore.dataproduct_subtype" description: Data product specific type nullable: true @@ -286,8 +286,8 @@ tables: votable:utype: ObsDataset.dataProductSubtype tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: target_class # (optional) + ivoa:unit: + - name: target_class # (optional) "@id": "ObsCore.target_class" description: Class of the Target object as in SSA nullable: true @@ -295,8 +295,8 @@ tables: votable:utype: Target.class tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: obs_creation_date # (optional) + ivoa:unit: + - name: obs_creation_date # (optional) "@id": "ObsCore.obs_creation_date" description: Date when the dataset was created nullable: true @@ -304,8 +304,8 @@ tables: votable:utype: DataID.date tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: obs_creator_name # (optional) + ivoa:unit: + - name: obs_creator_name # (optional) "@id": "ObsCore.obs_creator_name" description: Name of the creator of the data nullable: true @@ -313,8 +313,8 @@ tables: votable:utype: DataID.creator tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: obs_creator_did # (optional) + ivoa:unit: + - name: obs_creator_did # (optional) "@id": "ObsCore.obs_creator_did" description: IVOA dataset identifier given by the creator nullable: true @@ -322,8 +322,8 @@ tables: votable:utype: DataID.creatorDID tap:std: 1 tap:principal: 0 - ivoa:unit: - - name: obs_title # (optional) + ivoa:unit: + - name: obs_title # (optional) "@id": "ObsCore.obs_title" description: Brief description of dataset in free format nullable: true @@ -331,8 +331,8 @@ tables: votable:utype: DataID.title tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: publisher_id # (optional) + ivoa:unit: + - name: publisher_id # (optional) "@id": "ObsCore.publisher_id" description: IVOA-ID for the Publisher nullable: true @@ -340,8 +340,8 @@ tables: votable:utype: Curation.publisherID tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: bib_reference # (optional) + ivoa:unit: + - name: bib_reference # (optional) "@id": "ObsCore.bib_reference" description: Service bibliographic reference nullable: true @@ -349,8 +349,8 @@ tables: votable:utype: Curation.reference tap:std: 1 tap:principal: 0 - ivoa:unit: - - name: data_rights # (optional) + ivoa:unit: + - name: data_rights # (optional) "@id": "ObsCore.data_rights" description: Public/Secure/Proprietary/ nullable: true @@ -358,8 +358,8 @@ tables: votable:utype: Curation.rights tap:std: 1 tap:principal: 0 - ivoa:unit: - - name: obs_release_date # (optional) + ivoa:unit: + - name: obs_release_date # (optional) "@id": "ObsCore.obs_release_date" description: Observation release date (ISO 8601) nullable: true @@ -367,8 +367,8 @@ tables: votable:utype: Curation.releaseDate tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: s_ucd # (optional) + ivoa:unit: + - name: s_ucd # (optional) "@id": "ObsCore.s_ucd" description: UCD for the nature of the spatial axis (pos or u,v data) nullable: true @@ -376,8 +376,8 @@ tables: votable:utype: Char.SpatialAxis.ucd tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: s_unit # (optional) + ivoa:unit: + - name: s_unit # (optional) "@id": "ObsCore.s_unit" description: Unit used for spatial axis nullable: true @@ -385,8 +385,8 @@ tables: votable:utype: Char.SpatialAxis.unit tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: s_resolution_min # (optional) + ivoa:unit: + - name: s_resolution_min # (optional) "@id": "ObsCore.s_resolution_min" description: Resolution min value on spatial axis (FHWM of PSF) nullable: true @@ -395,7 +395,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: arcsec - - name: s_resolution_max # (optional) + - name: s_resolution_max # (optional) "@id": "ObsCore.s_resolution_max" description: Resolution max value on spatial axis nullable: true @@ -404,7 +404,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: arcsec - - name: s_pixel_scale # (optional) + - name: s_pixel_scale # (optional) "@id": "ObsCore.s_pixel_scale" description: Sampling period in world coordinate units along the spatial axis nullable: true @@ -413,7 +413,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: arcsec - - name: s_calib_status # (optional) + - name: s_calib_status # (optional) "@id": "ObsCore.s_calib_status" description: Type of calibration along the spatial axis nullable: true @@ -421,8 +421,8 @@ tables: votable:utype: Char.SpatialAxis.calibrationStatus tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: s_stat_error # (optional) + ivoa:unit: + - name: s_stat_error # (optional) "@id": "ObsCore.s_stat_error" description: Astrometric precision along the spatial axis nullable: true @@ -431,7 +431,7 @@ tables: tap:std: 1 tap:principal: 0 ivoa:unit: arcsec - - name: t_calib_status # (optional) + - name: t_calib_status # (optional) "@id": "ObsCore.t_calib_status" description: Type of time coordinate calibration nullable: true @@ -439,8 +439,8 @@ tables: votable:utype: Char.TimeAxis.calibrationStatus tap:std: 1 tap:principal: 0 - ivoa:unit: - - name: t_stat_error # (optional) + ivoa:unit: + - name: t_stat_error # (optional) "@id": "ObsCore.t_stat_error" description: Time coord statistical error nullable: true @@ -449,7 +449,7 @@ tables: tap:std: 1 tap:principal: 0 ivoa:unit: s - - name: em_ucd # (optional) + - name: em_ucd # (optional) "@id": "ObsCore.em_ucd" description: Nature of the spectral axis nullable: true @@ -457,8 +457,8 @@ tables: votable:utype: Char.SpectralAxis.ucd tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: em_unit # (optional) + ivoa:unit: + - name: em_unit # (optional) "@id": "ObsCore.em_unit" description: Units along the spectral axis nullable: true @@ -466,8 +466,8 @@ tables: votable:utype: Char.SpectralAxis.unit tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: em_calib_status # (optional) + ivoa:unit: + - name: em_calib_status # (optional) "@id": "ObsCore.em_calib_status" description: Type of spectral coord calibration nullable: true @@ -475,8 +475,8 @@ tables: votable:utype: Char.SpectralAxis.calibrationStatus tap:std: 1 tap:principal: 0 - ivoa:unit: - - name: em_res_power_min # (optional) + ivoa:unit: + - name: em_res_power_min # (optional) "@id": "ObsCore.em_res_power_min" description: Resolving power min value on spectral axis nullable: true @@ -484,8 +484,8 @@ tables: votable:utype: Char.SpectralAxis.Resolution.ResolPower.LoLimit tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: em_res_power_max # (optional) + ivoa:unit: + - name: em_res_power_max # (optional) "@id": "ObsCore.em_res_power_max" description: Resolving power max value on spectral axis nullable: true @@ -493,8 +493,8 @@ tables: votable:utype: Char.SpectralAxis.Resolution.ResolPower.HiLimit tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: em_resolution # (optional) + ivoa:unit: + - name: em_resolution # (optional) "@id": "ObsCore.em_resolution" description: Value of Resolution along the spectral axis nullable: true @@ -503,7 +503,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: m - - name: em_stat_error # (optional) + - name: em_stat_error # (optional) "@id": "ObsCore.em_stat_error" description: Spectral coord statistical error nullable: true @@ -512,7 +512,7 @@ tables: tap:std: 1 tap:principal: 0 ivoa:unit: m - - name: o_unit # (optional) + - name: o_unit # (optional) "@id": "ObsCore.o_unit" description: Units used for the observable values nullable: true @@ -520,8 +520,8 @@ tables: votable:utype: Char.ObservableAxis.unit tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: o_calib_status # (optional) + ivoa:unit: + - name: o_calib_status # (optional) "@id": "ObsCore.o_calib_status" description: Type of calibration for the observable coordinate nullable: true @@ -529,8 +529,8 @@ tables: votable:utype: Char.ObservableAxis.calibrationStatus tap:std: 1 tap:principal: 1 - ivoa:unit: - - name: o_stat_error # (optional) + ivoa:unit: + - name: o_stat_error # (optional) "@id": "ObsCore.o_stat_error" description: Statistical error on the Observable axis nullable: true @@ -539,7 +539,7 @@ tables: tap:std: 1 tap:principal: 0 ivoa:unit: same units as the value of the o_unit attribute - - name: proposal_id # (optional) + - name: proposal_id # (optional) "@id": "ObsCore.proposal_id" description: Identifier of proposal to which observation belongs nullable: true @@ -547,4 +547,4 @@ tables: votable:utype: Provenance.Proposal.identifier tap:std: 1 tap:principal: 0 - ivoa:unit: + ivoa:unit: From 706b696528e534c9fa841c8eec4efbf772a3e3a1 Mon Sep 17 00:00:00 2001 From: Fritz Mueller Date: Fri, 27 May 2022 13:28:21 -0700 Subject: [PATCH 08/10] Add dp02_obscore.yaml to idfint container --- tap-schema/build-all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap-schema/build-all b/tap-schema/build-all index 07475177..c2fff644 100755 --- a/tap-schema/build-all +++ b/tap-schema/build-all @@ -3,4 +3,4 @@ ./build stable ../yml/hsc.yaml ../yml/wise_01.yml ../yml/dp01_dc2.yaml ../yml/dp02_dc2_preops-863.yaml ./build int ../yml/hsc.yaml ../yml/wise_01.yml ../yml/dp01_dc2.yaml ../yml/dp02_dc2_preops-863.yaml ./build idfprod ../yml/dp01_dc2.yaml -./build idfint ../yml/dp01_dc2.yaml ../yml/dp02_dc2_preops-863.yaml ../yml/dp02_dc2.yaml +./build idfint ../yml/dp01_dc2.yaml ../yml/dp02_dc2_preops-863.yaml ../yml/dp02_dc2.yaml ../yml/dp02_obscore.yaml From 37dd9245a85c9843781827c1a2cf0e2f01db1fbe Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Fri, 27 May 2022 14:19:34 -0700 Subject: [PATCH 09/10] Add partial support for datatype sizes --- yml/dp02_obscore.yaml | 43 +++++++++++++++++++++++++++++++- yml/ivoa/make_obscore_nominal.py | 39 +++++++++++++++++++++++------ 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/yml/dp02_obscore.yaml b/yml/dp02_obscore.yaml index 296f4423..75dce1b9 100644 --- a/yml/dp02_obscore.yaml +++ b/yml/dp02_obscore.yaml @@ -3,7 +3,7 @@ name: ivoa "@id": "#ivoa_obscore" description: ObsCore v1.1 attributes in ObsTAP realization tables: -- name: ivoa.ObsCore +- name: ObsCore "@id": "#ObsCore" description: Observation metadata in the ObsTAP relational realization of the IVOA ObsCore data model @@ -17,6 +17,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: dataproduct_subtype "@id": "ObsCore.dataproduct_subtype" description: Data product specific type @@ -26,6 +28,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: facility_name "@id": "ObsCore.facility_name" description: The name of the facility, telescope, or space craft used for the observation @@ -35,6 +39,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: calib_level "@id": "ObsCore.calib_level" description: "Calibration level of the observation: in {0, 1, 2, 3, 4}" @@ -44,6 +50,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: int - name: target_name "@id": "ObsCore.target_name" description: Object of interest @@ -53,6 +60,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: obs_id "@id": "ObsCore.obs_id" description: Internal ID given by the ObsTAP service @@ -62,6 +71,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: obs_collection "@id": "ObsCore.obs_collection" description: Name of the data collection @@ -71,6 +82,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: obs_publisher_did "@id": "ObsCore.obs_publisher_did" description: ID for the Dataset given by the publisher @@ -80,6 +93,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: access_url "@id": "ObsCore.access_url" description: URL used to access dataset @@ -89,6 +104,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: access_format "@id": "ObsCore.access_format" description: Content format of the dataset @@ -98,6 +115,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: s_ra "@id": "ObsCore.s_ra" description: Central Spatial Position in ICRS; Right ascension @@ -107,6 +126,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: deg + datatype: double - name: s_dec "@id": "ObsCore.s_dec" description: Central Spatial Position in ICRS; Declination @@ -116,6 +136,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: deg + datatype: double - name: s_fov "@id": "ObsCore.s_fov" description: Estimated size of the covered region as the diameter of a containing circle @@ -125,6 +146,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: deg + datatype: double - name: s_region "@id": "ObsCore.s_region" description: Sky region covered by the data product (expressed in ICRS frame) @@ -134,6 +156,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: s_resolution "@id": "ObsCore.s_resolution" description: Spatial resolution of data as FWHM of PSF @@ -143,6 +167,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: arcsec + datatype: double - name: s_xel1 "@id": "ObsCore.s_xel1" description: Number of elements along the first coordinate of the spatial axis @@ -152,6 +177,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: long - name: s_xel2 "@id": "ObsCore.s_xel2" description: Number of elements along the second coordinate of the spatial axis @@ -161,6 +187,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: long - name: t_xel "@id": "ObsCore.t_xel" description: Number of elements along the time axis @@ -170,6 +197,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: long - name: t_min "@id": "ObsCore.t_min" description: Start time in MJD @@ -179,6 +207,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: d + datatype: double - name: t_max "@id": "ObsCore.t_max" description: Stop time in MJD @@ -188,6 +217,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: d + datatype: double - name: t_exptime "@id": "ObsCore.t_exptime" description: Total exposure time @@ -197,6 +227,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: s + datatype: double - name: t_resolution "@id": "ObsCore.t_resolution" description: Temporal resolution FWHM @@ -206,6 +237,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: s + datatype: double - name: em_xel "@id": "ObsCore.em_xel" description: Number of elements along the spectral axis @@ -215,6 +247,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: long - name: em_min "@id": "ObsCore.em_min" description: start in spectral coordinates @@ -224,6 +257,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: m + datatype: double - name: em_max "@id": "ObsCore.em_max" description: stop in spectral coordinates @@ -233,6 +267,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: m + datatype: double - name: em_res_power "@id": "ObsCore.em_res_power" description: Value of the resolving power along the spectral axis (R) @@ -242,6 +277,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: double - name: o_ucd "@id": "ObsCore.o_ucd" description: Nature of the observable axis @@ -251,6 +287,8 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" - name: pol_xel "@id": "ObsCore.pol_xel" description: Number of elements along the polarization axis @@ -260,6 +298,7 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: long - name: instrument_name "@id": "ObsCore.instrument_name" description: The name of the instrument used for the observation @@ -269,3 +308,5 @@ tables: tap:std: 1 tap:principal: 1 ivoa:unit: + datatype: char + votable:arraysize: "*" diff --git a/yml/ivoa/make_obscore_nominal.py b/yml/ivoa/make_obscore_nominal.py index fad5f051..778c2e66 100644 --- a/yml/ivoa/make_obscore_nominal.py +++ b/yml/ivoa/make_obscore_nominal.py @@ -116,13 +116,22 @@ def to_yaml( text: str ) -> str: #TODO - validate the list to ensure it contains only known attributes, # and all the mandatory ones. + # Add a Felis-compatible datatype mapping from the types provided in the Table 5 data + # TODO - this doesn't handle all cases yet, especially "date" + felis_types = { "enum int": ("int", 1), + "integer": ("long", 1), + "enum string": ("char", '"*"'), + "string": ("char", '"*"'), + "String": ("char", '"*"'), + "double": ("double", 1) } + # emit file-level header describing the file and the single table it defines print("---\n" "name: ivoa\n" "\"@id\": \"#ivoa_obscore\"\n" "description: ObsCore v1.1 attributes in ObsTAP realization\n" "tables:\n" - "- name: ivoa.ObsCore\n" + "- name: ObsCore\n" " \"@id\": \"#ObsCore\"\n" " description: Observation metadata in the ObsTAP relational realization of\n" " the IVOA ObsCore data model\n" @@ -136,7 +145,7 @@ def to_yaml( text: str ) -> str: for col in col_list: print( " - name: " + col + - (" # (optional)" if mark_opt and col_data[col]['optional'] else "") + "\n" + (" # (optional)" if mark_opt and col_data[col]['optional'] else "") + "\n" " \"@id\": \"ObsCore." + col + "\"\n" " description: " + to_yaml(desc_data[col]['Description']) + "\n" " nullable: " + ( "false" if col in not_null else "true" ) + "\n" @@ -149,9 +158,23 @@ def to_yaml( text: str ) -> str: # In the ObsCore document, from which the CSV files were copied, columns with # suggested null values have the string "NULL", which we should not emit into Felis. col_units = col_data[col]['Units'] - if col_units == 'NULL': col_units = '' - print( - " ivoa:unit: " + col_units - ) - - + if col_units == 'NULL': + print( + " ivoa:unit:" + ) + else: + print( + " ivoa:unit: " + col_units + ) + + # Output the data type, which may be an array + ftype = felis_types[desc_data[col]["Type"]] + if ftype[1] == 1: + print( + " datatype: " + ftype[0] + ) + else: + print( + " datatype: " + ftype[0] + "\n" + " votable:arraysize: " + ftype[1] + ) From 17a467c452bc16d4912bbb5e5c1c8d660230a77c Mon Sep 17 00:00:00 2001 From: Gregory Dubois-Felsmann Date: Fri, 27 May 2022 15:01:50 -0700 Subject: [PATCH 10/10] Hand-edit array sizes for string columns. --- yml/dp02_obscore.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/yml/dp02_obscore.yaml b/yml/dp02_obscore.yaml index 75dce1b9..e12ad8e0 100644 --- a/yml/dp02_obscore.yaml +++ b/yml/dp02_obscore.yaml @@ -18,7 +18,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 128 - name: dataproduct_subtype "@id": "ObsCore.dataproduct_subtype" description: Data product specific type @@ -29,7 +29,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 64 - name: facility_name "@id": "ObsCore.facility_name" description: The name of the facility, telescope, or space craft used for the observation @@ -40,7 +40,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 128 - name: calib_level "@id": "ObsCore.calib_level" description: "Calibration level of the observation: in {0, 1, 2, 3, 4}" @@ -61,7 +61,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 32 - name: obs_id "@id": "ObsCore.obs_id" description: Internal ID given by the ObsTAP service @@ -72,7 +72,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 128 - name: obs_collection "@id": "ObsCore.obs_collection" description: Name of the data collection @@ -83,7 +83,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 128 - name: obs_publisher_did "@id": "ObsCore.obs_publisher_did" description: ID for the Dataset given by the publisher @@ -94,7 +94,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 256 - name: access_url "@id": "ObsCore.access_url" description: URL used to access dataset @@ -116,7 +116,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 128 - name: s_ra "@id": "ObsCore.s_ra" description: Central Spatial Position in ICRS; Right ascension @@ -157,7 +157,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 512 - name: s_resolution "@id": "ObsCore.s_resolution" description: Spatial resolution of data as FWHM of PSF @@ -288,7 +288,7 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 32 - name: pol_xel "@id": "ObsCore.pol_xel" description: Number of elements along the polarization axis @@ -309,4 +309,4 @@ tables: tap:principal: 1 ivoa:unit: datatype: char - votable:arraysize: "*" + votable:arraysize: 128