xyz_reduce_precision: forcing 8 digits (#1392)

* xyz_reduce_precision: forceing 8 digits Signed-off-by: qGYdXbY2 <[email protected]> * adj. test Signed-off-by: qGYdXbY2 <[email protected]> * tmp. solution - additional export only wkb with 8 digits Signed-off-by: qGYdXbY2 <[email protected]> --------- Signed-off-by: qGYdXbY2 <[email protected]>
heremaps · Nov 6, 2024 · 186a452 · 186a452
1 parent e3bf17a
commit 186a452
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 15 deletions.
diff --git a/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/config/JDBCExporter.java b/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/config/JDBCExporter.java
@@ -720,7 +720,7 @@ private SQLQuery generateFilteredExportQuery(JdbcClient client, String schema, S
           }
 
          case PARTITIONED_JSON_WKB :
-         case PARTITIONID_FC_B64   :
+         case PARTITIONID_FC_B64   : 
          {
             String partQry =
                          csvFormat == PARTITIONID_FC_B64
@@ -735,13 +735,14 @@ private SQLQuery generateFilteredExportQuery(JdbcClient client, String schema, S
                               + " replace( encode(convert_to(jsonb_build_object( 'type','FeatureCollection','features', jsonb_build_array( jsondata || jsonb_build_object( 'geometry', ST_AsGeoJSON(geo,8)::jsonb ) ) )::text,'UTF8'),'base64') ,chr(10),'') as data "
                               + "from ( ${{contentQuery}}) X" )
                        /* PARTITIONED_JSON_WKB */
+                       /*TODO:  "st_geomfromtext(st_astext(geo,8))" conversion is a tmp solution for export of wkb, should be removed on later releases when all geom in db are aligned to 8 digit prec. */
                             : ( isForCompositeContentDetection
                               ? "select jsondata->>'id' as id, "
                               + " case not coalesce((jsondata#>'{properties,@ns:com:here:xyz,deleted}')::boolean,false) when true then jsondata else null::jsonb end as jsondata,"
-                              + " geo "
+                              + " st_geomfromtext(st_astext(geo,8),4326) as geo "
                               + "from ( ${{contentQuery}}) X"
-                              : "select jsondata->>'id' as id, jsondata, geo "
-                              + "from ( ${{contentQuery}}) X" );
+                              : "select jsondata->>'id' as id, jsondata, st_geomfromtext(st_astext(geo,8),4326) as geo "
+                              + "from ( ${{contentQuery}}) X" );  
 
            if( partitionByPropertyValue )
            {
@@ -773,7 +774,7 @@ private SQLQuery generateFilteredExportQuery(JdbcClient client, String schema, S
                     +" ( select coalesce( ('[]'::jsonb || key)->>0, 'CSVNULL' ) as id, jsondata, geo "
                     +"   from iidata "
                     +" )   "
-                    +" select id, jsondata, geo from iiidata "
+                    +" select id, jsondata, st_geomfromtext(st_astext(geo,8),4326) as geo from iiidata "
                   );
            }
 
@@ -789,7 +790,8 @@ private SQLQuery generateFilteredExportQuery(JdbcClient client, String schema, S
             default:
             {
               // JSON_WKB, DOWNLOAD
-              contentQuery = new SQLQuery("SELECT jsondata, geo FROM (${{innerContentQuery}}) contentQuery")
+              /*TODO:  "st_geomfromtext(st_astext(geo,8))" conversion is a tmp solution for export of wkb, should be removed on later releases when all geom in db are aligned to 8 digit prec. */
+              contentQuery = new SQLQuery("SELECT jsondata, st_geomfromtext(st_astext(geo,8),4326) as geo FROM (${{innerContentQuery}}) contentQuery")
                   .withQueryFragment("innerContentQuery", contentQuery);
                 return queryToText(contentQuery,"vc" + cFlag);
             }

diff --git a/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/jobs/JobApiCompositeExportIT.java b/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/jobs/JobApiCompositeExportIT.java
@@ -63,7 +63,7 @@ public class JobApiCompositeExportIT extends JobApiIT{
             add("id5");
             add("id6");
             add("id7");
-            add("01010000A0E6100000F1F91F681EE23C40FCAA9EC4CEE939400000000000000000");
+            add("01010000A0E6100000F0F91F681EE23C40FBAA9EC4CEE939400000000000000000");
         }};
 
     private static List<String> l1ChangesContentTILEID_FC_B64 = new ArrayList<>(){

diff --git a/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/jobs/JobApiExportIT.java b/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/jobs/JobApiExportIT.java
@@ -65,7 +65,7 @@ public void testFullWKBExport() throws Exception {
         List<URL> urls = performExport(job, getScopedSpaceId(testSpaceId2, scope), finalized, failed);
 
         List<String> mustContain = Arrays.asList(
-            "01020000A0E61000000300000042C17BFDFF662140E08442041C14494000000000000000006BDE27FD732F21406C5CFFAECF0A49400000000000000000A9B7ABFCD7162140A96A82A8FB0E49400000000000000000",
+            "01020000A0E61000000300000041C17BFDFF662140E08442041C14494000000000000000006BDE27FD732F21406C5CFFAECF0A49400000000000000000A9B7ABFCD7162140A96A82A8FB0E49400000000000000000",
             "01030000A0E6100000010000000500000083E4F8FC8B17214074F04C689202494000000000000000004DF564FED1A3214074F04C689202494000000000000000004DF564FED1A32140876D8B321B184940000000000000000083E4F8FC8B172140876D8B321B184940000000000000000083E4F8FC8B17214074F04C68920249400000000000000000",
             "foo_polygon"
         );
@@ -93,7 +93,7 @@ public void testFilteredWKBExportWithSpatialFilterClipped() throws Exception {
 
         List<String> mustContain = Arrays.asList(
             "foo_polygon",
-            "01030000A0E61000000100000021000000DA81E05D2D7E21404E355FE7F70B49400000000000000000DE78CF5FF67B2140D4078B84C70A494000000000000000007F932C6359782140671DB2C2AE094940000000000000000058FB14157A73214019160768B8084940000000000000000093A77370886D21405E1C90E7ED0749400000000000000000C6A82FE4BE6621400654E3045707494000000000000000000415FE15605F2140A53D7888F906494000000000000000008B4C9B57B457214055905B07D90649400000000000000000DD61AFE6065021400D6A58C0F6064940000000000000000048B14F11A34821409417DE8F5107494000000000000000005360E959D141214006CD13FBE507494000000000000000002CBA64B5D43B2140D53CB251AE0849400000000000000000513B80FDE7362140F2D664E5A209494000000000000000007591BCAF3B332140B95FA854BA0A49400000000000000000CDA0BB0EF4302140BB2F61E6E90B49400000000000000000A718B2B8273021402E50BFF2250D494000000000000000000E6B89C1DE3021402B058055620E49400000000000000000FC77A35A1233214044E038E5920F494000000000000000000C98F80CAD362140990F1DEBAB1049400000000000000000B81DAE848B3B2140A2089E96A211494000000000000000008C8A7CE77D412140E42A67686D1249400000000000000000BA1D89A94948214051F18090041349400000000000000000682AFCCEAB4F2140F011E23B62134940000000000000000019C8E4835B572140F29357CE821349400000000000000000590827F00C5F2140C5447206651349400000000000000000020F7E2A7466214009DE070A0A134940000000000000000016D4312C486D2140E6D7C65A751249400000000000000000DFBF17A745732140407152B3AC11494000000000000000002B55CEA03178214034B34DCEB71049400000000000000000A010C9B8DB7B2140EA379D19A00F494000000000000000008FD98802207E21407169EE58700E4940000000000000000054590D62E87E21406979373B340D49400000000000000000DA81E05D2D7E21404E355FE7F70B49400000000000000000"
+            "01030000A0E610000001000000210000003629DC5D2D7E214081BD57E7F70B494000000000000000001D34C35FF67B21409A6F8E84C70A4940000000000000000097B11D6359782140891DB6C2AE094940000000000000000092ED19157A73214078E01068B80849400000000000000000C0765170886D2140E61A8FE7ED074940000000000000000027D756E4BE662140EB0CDF04570749400000000000000000104DEA15605F2140211A8288F906494000000000000000006EF4B557B4572140FC385807D90649400000000000000000680EA4E60650214088BD50C0F6064940000000000000000077866711A3482140AD1FD58F5107494000000000000000005947FA59D14121408A151CFBE50749400000000000000000EC804EB5D43B2140B982B751AE08494000000000000000002ABBAAFDE7362140713C5DE5A20949400000000000000000FF76D9AF3B332140DC1DA554BA0A49400000000000000000121CB00EF4302140E22063E6E90B4940000000000000000098ADABB8273021408156C3F2250D4940000000000000000094F06BC1DE30214003A18A55620E49400000000000000000977DCB5A1233214085F834E5920F494000000000000000009CF1E80CAD3621403D6215EBAB104940000000000000000042EDD8848B3B2140595D9896A21149400000000000000000B46B8CE77D412140C24063686D12494000000000000000009F3D97A949482140E07888900413494000000000000000004350FBCEAB4F21403954DF3B6213494000000000000000002548DF835B572140580651CE821349400000000000000000707EFDEF0C5F214073FC6906651349400000000000000000EE3A9F2A74662140B390040A0A134940000000000000000052A23E2C486D2140BEFCCA5A7512494000000000000000001E31FEA64573214084D151B3AC114940000000000000000040EFB6A03178214066804DCEB7104940000000000000000054E7CDB8DB7B214067EEA519A00F4940000000000000000079D3A902207E2140D202EE58700E49400000000000000000B3A2FE61E87E2140AA82303B340D494000000000000000003629DC5D2D7E214081BD57E7F70B49400000000000000000"
         );
 
         downloadAndCheck(urls, 3907, 5, mustContain);
@@ -114,7 +114,7 @@ public void testFilteredWKBExportWithSpatialFilterUnclipped() throws Exception {
 
         List<String> mustContain = Arrays.asList(
             "foo_polygon",
-            "01020000A0E610000004000000901FE0FF7D70214087A4164A260B4940000000000000000041237DFF8B4421402A7288B8390549400000000000000000407E80FFF70521400264E8D8410749400000000000000000F60E12FD350421401AA9F7544E0349400000000000000000"
+            "01020000A0E610000004000000901FE0FF7D70214087A4164A260B4940000000000000000041237DFF8B4421402A7288B8390549400000000000000000407E80FFF70521400264E8D8410749400000000000000000F60E12FD3504214019A9F7544E0349400000000000000000"
         );
 
         downloadAndCheck(urls, 2467, 5, mustContain);
@@ -133,7 +133,7 @@ public void testFilteredWKBExportWithPropFilter() throws Exception {
 
         List<String> mustContain = Arrays.asList(
             "onPropertyLevel",
-            "01010000A0E610000038E38FFD875721402A55A2EC2D0D49400000000000000000"
+            "01010000A0E610000037E38FFD875721402A55A2EC2D0D49400000000000000000"
         );
 
         downloadAndCheck(urls, 890, 3, mustContain);
@@ -157,7 +157,7 @@ public void testFilteredWKBExportWithPropAndSpatialFilter() throws Exception {
 
         List<String> mustContain = Arrays.asList(
             "Point",
-            "01010000A0E610000038E38FFD875721402A55A2EC2D0D49400000000000000000"
+            "01010000A0E610000037E38FFD875721402A55A2EC2D0D49400000000000000000"
         );
 
         downloadAndCheck(urls, 328, 1, mustContain);

diff --git a/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java b/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java
@@ -50,7 +50,7 @@ public class DatabaseMaintainer {
     private static final Logger logger = LogManager.getLogger();
 
     /** Is used to check against xyz_ext_version() */
-    public static final int XYZ_EXT_VERSION = 202;
+    public static final int XYZ_EXT_VERSION = 203;
 
     public static final int H3_CORE_VERSION = 108;
 

diff --git a/xyz-psql-connector/src/main/resources/xyz_ext.sql b/xyz-psql-connector/src/main/resources/xyz_ext.sql
@@ -110,7 +110,7 @@
 CREATE OR REPLACE FUNCTION xyz_ext_version()
   RETURNS integer AS
 $BODY$
- select 202
+ select 203
 $BODY$
   LANGUAGE sql IMMUTABLE;
 
@@ -129,7 +129,7 @@ BEGIN
    RETURN geo;
   end if;
 
-  sgeo := ST_SnapToGrid(geo, 0.00000001); -- ST_ReducePrecision(geo, 0.00000001);
+  sgeo := st_geomfromtext(st_astext(ST_SnapToGrid(geo, 0.00000001),8),4326); -- ST_ReducePrecision(geo, 0.00000001);
 
   IF GeometryType(sgeo) = GeometryType(geo) THEN
    RETURN sgeo;  -- only if type did not changed