From c50da559f717d035d6fd2f99d8b17762876db864 Mon Sep 17 00:00:00 2001
From: Jeremy McCormick <jeremym@slac.stanford.edu>
Date: Fri, 26 Apr 2024 13:49:31 -0500
Subject: [PATCH 1/2] Add missing lengths for columns with sized types

Felis will no longer accept columns with sized types that are missing
a length. All such columns were updated to include explicit length
values where they were missing.

Some columns had an existing `votable:arraysize` that was changed to a
`length`. This value will be used automatically for the `arraysize`
when generating the TAP_SCHEMA.

A few columns were changed from a sized string type to `text` where
the VOTable type had been specified already as `clob`.
---
 yml/dp02_obscore.yaml     | 27 ++++++++++++++-------------
 yml/imsim.yaml            |  2 +-
 yml/obsloctap.yaml        |  4 ++--
 yml/oga_live_obscore.yaml | 25 +++++++++++++------------
 4 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/yml/dp02_obscore.yaml b/yml/dp02_obscore.yaml
index c6b950a1..0f8dcecb 100644
--- a/yml/dp02_obscore.yaml
+++ b/yml/dp02_obscore.yaml
@@ -20,7 +20,7 @@ tables:
     tap:column_index: 10
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: dataproduct_subtype
     "@id": "ObsCore.dataproduct_subtype"
     description: Data product specific type
@@ -32,7 +32,7 @@ tables:
     tap:column_index: 20
     ivoa:unit:
     datatype: char
-    votable:arraysize: 64
+    length: 64
   - name: obs_title
     "@id": "ObsCore.obs_title"
     description: Brief description of dataset in free format
@@ -44,6 +44,7 @@ tables:
     tap:column_index: 225
     ivoa:unit:
     datatype: char
+    length: 255  # FIXME: Not clear what should be the length; ObsCore REC suggests not unbounded ("single line of text")
     votable:arraysize: "*"
   - name: facility_name
     "@id": "ObsCore.facility_name"
@@ -56,7 +57,7 @@ tables:
     tap:column_index: 210
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: calib_level
     "@id": "ObsCore.calib_level"
     description: "Calibration level of the observation: in {0, 1, 2, 3, 4}"
@@ -79,7 +80,7 @@ tables:
     tap:column_index: 270
     ivoa:unit:
     datatype: char
-    votable:arraysize: 32
+    length: 32
   - name: obs_id
     "@id": "ObsCore.obs_id"
     description: Internal ID given by the ObsTAP service
@@ -91,7 +92,7 @@ tables:
     tap:column_index: 180
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: obs_collection
     "@id": "ObsCore.obs_collection"
     description: Name of the data collection
@@ -103,7 +104,7 @@ tables:
     tap:column_index: 190
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: obs_publisher_did
     "@id": "ObsCore.obs_publisher_did"
     description: ID for the Dataset given by the publisher
@@ -115,7 +116,7 @@ tables:
     tap:column_index: 260
     ivoa:unit:
     datatype: char
-    votable:arraysize: 256
+    length: 256
   - name: access_url
     "@id": "ObsCore.access_url"
     description: URL used to access dataset
@@ -126,7 +127,7 @@ tables:
     tap:principal: 1
     tap:column_index: 240
     ivoa:unit:
-    datatype: char
+    datatype: text
     votable:arraysize: "*"
     votable:xtype: clob
   - name: access_format
@@ -140,7 +141,7 @@ tables:
     tap:column_index: 250
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: s_ra
     "@id": "ObsCore.s_ra"
     description: Central Spatial Position in ICRS; Right ascension
@@ -184,8 +185,8 @@ tables:
     tap:principal: 1
     tap:column_index: 230
     ivoa:unit:
-    datatype: char
-    votable:arraysize: 512
+    datatype: string
+    length: 512
   - name: s_resolution
     "@id": "ObsCore.s_resolution"
     description: Spatial resolution of data as FWHM of PSF
@@ -329,7 +330,7 @@ tables:
     tap:column_index: 200
     ivoa:unit:
     datatype: char
-    votable:arraysize: 32
+    length: 32
   - name: pol_xel
     "@id": "ObsCore.pol_xel"
     description: Number of elements along the polarization axis
@@ -352,7 +353,7 @@ tables:
     tap:column_index: 220
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: lsst_visit
     "@id": "ObsCore.lsst_visit"
     description: Identifier for a specific LSSTCam pointing
diff --git a/yml/imsim.yaml b/yml/imsim.yaml
index a53e962a..660c9f43 100644
--- a/yml/imsim.yaml
+++ b/yml/imsim.yaml
@@ -8031,7 +8031,7 @@ tables:
   - name: band
     "@id": "#ForcedSourceOnDiaObject.band"
     datatype: char
-    mysql:datatype: CHAR(1)
+    length: 1
     description: Abstract filter that is not associated with a particular instrument
     fits:tunit:
   - name: ccdVisitId
diff --git a/yml/obsloctap.yaml b/yml/obsloctap.yaml
index 4b2e9940..6b0aee4c 100644
--- a/yml/obsloctap.yaml
+++ b/yml/obsloctap.yaml
@@ -215,9 +215,9 @@ tables:
         tap:principal: 0
         tap:column_index: 17
         ivoa:unit:
-        datatype: string
-        votable:arraysize: "*"
+        datatype: char
         length: 40
+        votable:arraysize: "*"
       - name: pol_states
         "@id": "#ObsPlan.pol_states"
         description: List of polarization states or NULL if not applicable
diff --git a/yml/oga_live_obscore.yaml b/yml/oga_live_obscore.yaml
index 3c2a5013..8dc8762b 100644
--- a/yml/oga_live_obscore.yaml
+++ b/yml/oga_live_obscore.yaml
@@ -20,7 +20,7 @@ tables:
     tap:column_index: 10
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: dataproduct_subtype
     "@id": "ObsCore.dataproduct_subtype"
     description: Data product specific type
@@ -32,7 +32,7 @@ tables:
     tap:column_index: 20
     ivoa:unit:
     datatype: char
-    votable:arraysize: 64
+    length: 64
   - name: facility_name
     "@id": "ObsCore.facility_name"
     description: The name of the facility, telescope, or space craft used for the observation
@@ -44,7 +44,7 @@ tables:
     tap:column_index: 210
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: calib_level
     "@id": "ObsCore.calib_level"
     description: "Calibration level of the observation: in {0, 1, 2, 3, 4}"
@@ -67,7 +67,7 @@ tables:
     tap:column_index: 270
     ivoa:unit:
     datatype: char
-    votable:arraysize: 32
+    length: 32
   - name: obs_id
     "@id": "ObsCore.obs_id"
     description: Internal ID given by the ObsTAP service
@@ -79,7 +79,7 @@ tables:
     tap:column_index: 180
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: obs_collection
     "@id": "ObsCore.obs_collection"
     description: Name of the data collection
@@ -91,7 +91,7 @@ tables:
     tap:column_index: 190
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: obs_publisher_did
     "@id": "ObsCore.obs_publisher_did"
     description: ID for the Dataset given by the publisher
@@ -103,7 +103,7 @@ tables:
     tap:column_index: 260
     ivoa:unit:
     datatype: char
-    votable:arraysize: 256
+    length: 256
   - name: access_url
     "@id": "ObsCore.access_url"
     description: URL used to access dataset
@@ -114,7 +114,7 @@ tables:
     tap:principal: 1
     tap:column_index: 240
     ivoa:unit:
-    datatype: char
+    datatype: text
     votable:arraysize: "*"
     votable:xtype: clob
   - name: access_format
@@ -128,7 +128,7 @@ tables:
     tap:column_index: 250
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: s_ra
     "@id": "ObsCore.s_ra"
     description: Central Spatial Position in ICRS; Right ascension
@@ -172,7 +172,8 @@ tables:
     tap:principal: 1
     tap:column_index: 230
     ivoa:unit:
-    datatype: char
+    datatype: string
+    length: 512
     votable:arraysize: 512
   - name: s_resolution
     "@id": "ObsCore.s_resolution"
@@ -317,7 +318,7 @@ tables:
     tap:column_index: 200
     ivoa:unit:
     datatype: char
-    votable:arraysize: 32
+    length: 32
   - name: pol_xel
     "@id": "ObsCore.pol_xel"
     description: Number of elements along the polarization axis
@@ -340,7 +341,7 @@ tables:
     tap:column_index: 220
     ivoa:unit:
     datatype: char
-    votable:arraysize: 128
+    length: 128
   - name: lsst_visit
     "@id": "ObsCore.lsst_visit"
     description: Identifier for a specific LSSTCam pointing

From f8d6ad5f4fdc105a29670a70908b453f770ad15b Mon Sep 17 00:00:00 2001
From: Jeremy McCormick <jeremym@slac.stanford.edu>
Date: Tue, 30 Apr 2024 14:58:45 -0500
Subject: [PATCH 2/2] Change char columns to string datatype in ObsCore columns

The live ObsCore database in MySQL uses VARCHAR columns, not CHAR.
---
 yml/dp02_obscore.yaml     | 28 ++++++++++++++--------------
 yml/obsloctap.yaml        |  2 +-
 yml/oga_live_obscore.yaml | 25 ++++++++++++-------------
 3 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/yml/dp02_obscore.yaml b/yml/dp02_obscore.yaml
index 0f8dcecb..9a18717c 100644
--- a/yml/dp02_obscore.yaml
+++ b/yml/dp02_obscore.yaml
@@ -19,7 +19,7 @@ tables:
     tap:principal: 1
     tap:column_index: 10
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: dataproduct_subtype
     "@id": "ObsCore.dataproduct_subtype"
@@ -31,7 +31,7 @@ tables:
     tap:principal: 1
     tap:column_index: 20
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 64
   - name: obs_title
     "@id": "ObsCore.obs_title"
@@ -43,8 +43,8 @@ tables:
     tap:principal: 1
     tap:column_index: 225
     ivoa:unit:
-    datatype: char
-    length: 255  # FIXME: Not clear what should be the length; ObsCore REC suggests not unbounded ("single line of text")
+    datatype: string
+    length: 256
     votable:arraysize: "*"
   - name: facility_name
     "@id": "ObsCore.facility_name"
@@ -56,7 +56,7 @@ tables:
     tap:principal: 1
     tap:column_index: 210
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: calib_level
     "@id": "ObsCore.calib_level"
@@ -79,7 +79,7 @@ tables:
     tap:principal: 0
     tap:column_index: 270
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 32
   - name: obs_id
     "@id": "ObsCore.obs_id"
@@ -91,7 +91,7 @@ tables:
     tap:principal: 1
     tap:column_index: 180
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: obs_collection
     "@id": "ObsCore.obs_collection"
@@ -103,7 +103,7 @@ tables:
     tap:principal: 1
     tap:column_index: 190
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: obs_publisher_did
     "@id": "ObsCore.obs_publisher_did"
@@ -115,7 +115,7 @@ tables:
     tap:principal: 0
     tap:column_index: 260
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 256
   - name: access_url
     "@id": "ObsCore.access_url"
@@ -140,7 +140,7 @@ tables:
     tap:principal: 1
     tap:column_index: 250
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: s_ra
     "@id": "ObsCore.s_ra"
@@ -329,7 +329,7 @@ tables:
     tap:principal: 1
     tap:column_index: 200
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 32
   - name: pol_xel
     "@id": "ObsCore.pol_xel"
@@ -352,7 +352,7 @@ tables:
     tap:principal: 1
     tap:column_index: 220
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: lsst_visit
     "@id": "ObsCore.lsst_visit"
@@ -419,7 +419,7 @@ tables:
     tap:principal: 1
     tap:column_index: 40
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 10
     votable:arraysize: 10
   - name: lsst_filter
@@ -432,6 +432,6 @@ tables:
     tap:principal: 1
     tap:column_index: 90
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 10
     votable:arraysize: 10
diff --git a/yml/obsloctap.yaml b/yml/obsloctap.yaml
index 6b0aee4c..4662004c 100644
--- a/yml/obsloctap.yaml
+++ b/yml/obsloctap.yaml
@@ -215,7 +215,7 @@ tables:
         tap:principal: 0
         tap:column_index: 17
         ivoa:unit:
-        datatype: char
+        datatype: string
         length: 40
         votable:arraysize: "*"
       - name: pol_states
diff --git a/yml/oga_live_obscore.yaml b/yml/oga_live_obscore.yaml
index 8dc8762b..a0cd8939 100644
--- a/yml/oga_live_obscore.yaml
+++ b/yml/oga_live_obscore.yaml
@@ -19,7 +19,7 @@ tables:
     tap:principal: 1
     tap:column_index: 10
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: dataproduct_subtype
     "@id": "ObsCore.dataproduct_subtype"
@@ -31,7 +31,7 @@ tables:
     tap:principal: 1
     tap:column_index: 20
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 64
   - name: facility_name
     "@id": "ObsCore.facility_name"
@@ -43,7 +43,7 @@ tables:
     tap:principal: 1
     tap:column_index: 210
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: calib_level
     "@id": "ObsCore.calib_level"
@@ -66,7 +66,7 @@ tables:
     tap:principal: 0
     tap:column_index: 270
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 32
   - name: obs_id
     "@id": "ObsCore.obs_id"
@@ -78,7 +78,7 @@ tables:
     tap:principal: 1
     tap:column_index: 180
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: obs_collection
     "@id": "ObsCore.obs_collection"
@@ -90,7 +90,7 @@ tables:
     tap:principal: 1
     tap:column_index: 190
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: obs_publisher_did
     "@id": "ObsCore.obs_publisher_did"
@@ -102,7 +102,7 @@ tables:
     tap:principal: 0
     tap:column_index: 260
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 256
   - name: access_url
     "@id": "ObsCore.access_url"
@@ -127,7 +127,7 @@ tables:
     tap:principal: 1
     tap:column_index: 250
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: s_ra
     "@id": "ObsCore.s_ra"
@@ -174,7 +174,6 @@ tables:
     ivoa:unit:
     datatype: string
     length: 512
-    votable:arraysize: 512
   - name: s_resolution
     "@id": "ObsCore.s_resolution"
     description: Spatial resolution of data as FWHM of PSF
@@ -317,7 +316,7 @@ tables:
     tap:principal: 1
     tap:column_index: 200
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 32
   - name: pol_xel
     "@id": "ObsCore.pol_xel"
@@ -340,7 +339,7 @@ tables:
     tap:principal: 1
     tap:column_index: 220
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 128
   - name: lsst_visit
     "@id": "ObsCore.lsst_visit"
@@ -396,7 +395,7 @@ tables:
     tap:principal: 1
     tap:column_index: 40
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 10
     votable:arraysize: 10
   - name: lsst_filter
@@ -409,6 +408,6 @@ tables:
     tap:principal: 1
     tap:column_index: 90
     ivoa:unit:
-    datatype: char
+    datatype: string
     length: 10
     votable:arraysize: 10