Merge pull request #420 from OpenDataServices/419-wkt

 flatten and unflatten: WKT <-> geojson conversion

.github/workflows/test.yml

@@ -3,11 +3,10 @@ on: [push, pull_request]
 
 jobs:
   build:
-    # Need to use an older Ubuntu so Python 3.6 is available
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11']
         jsonref-version: ["==0.3", ">1"]
     steps:
     - uses: actions/checkout@v2

.gitignore

@@ -60,4 +60,6 @@ examples/titles-ref/flatten/actual
 examples/titles-ref/flatten/actual.*
 examples/titles-ref/unflatten/actual
 examples/titles-ref/unflatten/actual.*
+examples/wkt/*/actual
+examples/wkt/*/actual.*
 .~lock*

CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [0.21.0] - 2023-06-23
+
+### Added
+
+- WKT <-> geojson conversion for flattening and unflattening, behind an optional flag https://github.com/OpenDataServices/flatten-tool/issues/419
+
+### Removed
+
+- We no longer support Python 3.6
+
 ## [0.20.1] - 2023-01-11
 
 ### Fixed

examples/help/flatten/expected.txt

@@ -10,6 +10,7 @@ usage: flatten-tool flatten [-h] [-s SCHEMA] [-f {csv,ods,xlsx,all}] [--xml]
                             [--disable-local-refs]
                             [--remove-empty-schema-columns]
                             [--line-terminator LINE_TERMINATOR]
+                            [--convert-wkt]
                             input_name
 
 positional arguments:
@@ -65,3 +66,4 @@ optional arguments:
   --line-terminator LINE_TERMINATOR
                         The line terminator to use when writing CSV files:
                         CRLF or LF
+  --convert-wkt         Enable conversion of geojson to WKT

examples/help/unflatten/expected.txt

@@ -13,7 +13,7 @@ usage: flatten-tool unflatten [-h] -f {csv,ods,xlsx} [--xml]
                               [--xml-schema [XML_SCHEMA [XML_SCHEMA ...]]]
                               [--default-configuration DEFAULT_CONFIGURATION]
                               [--root-is-list] [--disable-local-refs]
-                              [--xml-comment XML_COMMENT]
+                              [--xml-comment XML_COMMENT] [--convert-wkt]
                               input_name
 
 positional arguments:
@@ -75,3 +75,4 @@ optional arguments:
   --disable-local-refs  Disable local refs when parsing JSON Schema.
   --xml-comment XML_COMMENT
                         String comment of what generates the xml file
+  --convert-wkt         Enable conversion of WKT to geojson

examples/wkt/flatten/cmd.txt

@@ -0,0 +1 @@
+$ flatten-tool flatten -f=csv examples/wkt/flatten/data.json -o examples/wkt/flatten/actual --convert-wkt

examples/wkt/flatten/data.json

@@ -0,0 +1,10 @@
+{
+    "main": [
+        {
+            "geo": {
+                "type": "Point",
+                "coordinates": [53.486434, -2.239353]
+            }
+        }
+    ]
+}

examples/wkt/flatten/expected/main.csv

@@ -0,0 +1,2 @@
+geo
+POINT (53.486434 -2.239353)

examples/wkt/flatten_wkt_disabled/cmd.txt

@@ -0,0 +1 @@
+$ flatten-tool flatten -f=csv examples/wkt/flatten_wkt_disabled/data.json -o examples/wkt/flatten_wkt_disabled/actual

examples/wkt/flatten_wkt_disabled/data.json

@@ -0,0 +1,10 @@
+{
+    "main": [
+        {
+            "geo": {
+                "type": "Point",
+                "coordinates": [53.486434, -2.239353]
+            }
+        }
+    ]
+}

examples/wkt/flatten_wkt_disabled/expected/main.csv

@@ -0,0 +1,2 @@
+geo/type,geo/coordinates
+Point,53.486434;-2.239353

examples/wkt/unflatten/cmd.txt

@@ -0,0 +1 @@
+$ flatten-tool unflatten -f=csv examples/wkt/unflatten/ --schema examples/wkt/unflatten/schema.json --convert-wkt

examples/wkt/unflatten/expected.json

@@ -0,0 +1,13 @@
+{
+    "main": [
+        {
+            "geo": {
+                "type": "Point",
+                "coordinates": [
+                    53.486434,
+                    -2.239353
+                ]
+            }
+        }
+    ]
+}

examples/wkt/unflatten/main.csv

@@ -0,0 +1,2 @@
+geo
+POINT (53.486434 -2.239353)

examples/wkt/unflatten/schema.json

@@ -0,0 +1,11 @@
+{
+    "properties": {
+        "geo": {
+            "type": "object",
+            "properties": {
+                "type": {},
+                "coordinates": {}
+            }
+        }
+    }
+}

examples/wkt/unflatten_wkt_disabled/cmd.txt

@@ -0,0 +1 @@
+$ flatten-tool unflatten -f=csv examples/wkt/unflatten_wkt_disabled/ --schema examples/wkt/unflatten_wkt_disabled/schema.json

examples/wkt/unflatten_wkt_disabled/expected.json

@@ -0,0 +1,14 @@
+{
+    "main": [
+        {
+            "geo_string": "POINT (53.486434 -2.239353)",
+            "geo": {
+                "type": "Point",
+                "coordinates": [
+                    53.486434,
+                    -2.239353
+                ]
+            }
+        }
+    ]
+}

examples/wkt/unflatten_wkt_disabled/main.csv

@@ -0,0 +1,2 @@
+geo_string,geo/type,geo/coordinates
+POINT (53.486434 -2.239353),Point,53.486434;-2.239353

examples/wkt/unflatten_wkt_disabled/schema.json

@@ -0,0 +1,19 @@
+{
+    "properties": {
+        "geo": {
+            "type": "object",
+            "properties": {
+                "type": {},
+                "coordinates": {
+                    "type": "array",
+                    "items": {
+                        "type": "number"
+                    }
+                }
+            }
+        },
+        "geo_string": {
+            "type": "string"
+        }
+    }
+}

flattentool/__init__.py

@@ -95,6 +95,7 @@ def flatten(
     remove_empty_schema_columns=False,
     truncation_length=3,
     line_terminator="CRLF",
+    convert_wkt=False,
     **_,
 ):
     """
@@ -110,6 +111,8 @@ def flatten(
     if line_terminator not in LINE_TERMINATORS.keys():
         raise Exception(f"{line_terminator} is not a valid line terminator")
 
+    convert_flags = {"wkt": convert_wkt}
+
     if schema:
         schema_parser = SchemaParser(
             schema_filename=schema,
@@ -139,6 +142,7 @@ def flatten(
         remove_empty_schema_columns=remove_empty_schema_columns,
         truncation_length=truncation_length,
         persist=True,
+        convert_flags=convert_flags,
     ) as parser:
 
         def spreadsheet_output(spreadsheet_output_class, name):
@@ -218,19 +222,23 @@ def unflatten(
     disable_local_refs=False,
     xml_comment=None,
     truncation_length=3,
+    convert_wkt=False,
     **_,
 ):
     """
     Unflatten a flat structure (spreadsheet - csv or xlsx) into a nested structure (JSON).
 
     """
+
     if input_format is None:
         raise Exception("You must specify an input format (may autodetect in future")
     elif input_format not in INPUT_FORMATS:
         raise Exception("The requested format is not available")
     if metatab_name and base_json:
         raise Exception("Not allowed to use base_json with metatab")
 
+    convert_flags = {"wkt": convert_wkt}
+
     if root_is_list:
         base = None
     elif base_json:
@@ -258,6 +266,7 @@ def unflatten(
             id_name=id_name,
             xml=xml,
             use_configuration=False,
+            convert_flags=convert_flags,
         )
         if metatab_schema:
             parser = SchemaParser(
@@ -309,6 +318,7 @@ def unflatten(
             id_name=id_name,
             xml=xml,
             base_configuration=base_configuration,
+            convert_flags=convert_flags,
         )
         if schema:
             parser = SchemaParser(
@@ -317,6 +327,7 @@ def unflatten(
                 root_id=root_id,
                 disable_local_refs=disable_local_refs,
                 truncation_length=truncation_length,
+                convert_flags=convert_flags,
             )
             parser.parse()
             spreadsheet_input.parser = parser

flattentool/cli.py

@@ -185,6 +185,11 @@ def create_parser():
         "--line-terminator",
         help="The line terminator to use when writing CSV files: CRLF or LF",
     )
+    parser_flatten.add_argument(
+        "--convert-wkt",
+        action="store_true",
+        help="Enable conversion of geojson to WKT",
+    )
     parser_unflatten = subparsers.add_parser(
         "unflatten", help="Unflatten a spreadsheet"
     )
@@ -294,6 +299,11 @@ def create_parser():
         default="XML generated by flatten-tool",
         help="String comment of what generates the xml file",
     )
+    parser_unflatten.add_argument(
+        "--convert-wkt",
+        action="store_true",
+        help="Enable conversion of WKT to geojson",
+    )
 
     return parser
 

flattentool/input.py

@@ -13,8 +13,10 @@
 from decimal import Decimal, InvalidOperation
 from warnings import warn
 
+import geojson
 import openpyxl
 import pytz
+import shapely.wkt
 from openpyxl.utils.cell import _get_column_letter
 
 from flattentool.exceptions import DataErrorWarning
@@ -35,7 +37,7 @@ def __init__(self, cell_value, cell_location):
         self.sub_cells = []
 
 
-def convert_type(type_string, value, timezone=pytz.timezone("UTC")):
+def convert_type(type_string, value, timezone=pytz.timezone("UTC"), convert_flags={}):
     if value == "" or value is None:
         return None
     if type_string == "number":
@@ -103,6 +105,19 @@ def convert_type(type_string, value, timezone=pytz.timezone("UTC")):
         if type(value) == datetime.datetime:
             return value.date().isoformat()
         return str(value)
+    elif convert_flags.get("wkt") and type_string == "geojson":
+        try:
+            geom = shapely.wkt.loads(value)
+        except shapely.errors.GEOSException as e:
+            warn(
+                _(
+                    'An invalid WKT string was supplied "{value}", the message from the parser was: {parser_msg}'
+                ).format(value=value, parser_msg=str(e)),
+                DataErrorWarning,
+            )
+            return
+        feature = geojson.Feature(geometry=geom, properties={})
+        return feature.geometry
     elif type_string == "":
         if type(value) == datetime.datetime:
             return timezone.localize(value).isoformat()
@@ -258,6 +273,7 @@ def __init__(
         xml=False,
         base_configuration={},
         use_configuration=True,
+        convert_flags={},
     ):
         self.input_name = input_name
         self.root_list_path = root_list_path
@@ -275,6 +291,7 @@ def __init__(
         self.base_configuration = base_configuration or {}
         self.sheet_configuration = {}
         self.use_configuration = use_configuration
+        self.convert_flags = convert_flags
 
     def get_sub_sheets_lines(self):
         for sub_sheet_name in self.sub_sheet_names:
@@ -405,7 +422,12 @@ def do_unflatten(self):
                             (sheet_name, _get_column_letter(k + 1), j + 2, heading),
                         )
                 unflattened = unflatten_main_with_parser(
-                    self.parser, cells, self.timezone, self.xml, self.id_name
+                    self.parser,
+                    cells,
+                    self.timezone,
+                    self.xml,
+                    self.id_name,
+                    self.convert_flags,
                 )
                 if root_id_or_none not in main_sheet_by_ocid:
                     main_sheet_by_ocid[root_id_or_none] = TemporaryDict(
@@ -922,7 +944,7 @@ def list_as_dicts_to_temporary_dicts(unflattened, id_name, xml):
     return unflattened
 
 
-def unflatten_main_with_parser(parser, line, timezone, xml, id_name):
+def unflatten_main_with_parser(parser, line, timezone, xml, id_name, convert_flags={}):
     unflattened = OrderedDict()
     for path, cell in line.items():
         # Skip blank cells
@@ -1041,9 +1063,11 @@ def unflatten_main_with_parser(parser, line, timezone, xml, id_name):
                 # However the type of the text value itself should not be "array",
                 # as that would split the text on commas, which we don't want.
                 # https://github.com/OpenDataServices/cove/issues/1030
-                converted_value = convert_type("", value, timezone)
+                converted_value = convert_type("", value, timezone, convert_flags)
             else:
-                converted_value = convert_type(current_type or "", value, timezone)
+                converted_value = convert_type(
+                    current_type or "", value, timezone, convert_flags
+                )
             cell.cell_value = converted_value
             if converted_value is not None and converted_value != "":
                 if xml: