Merge pull request #300 from digital-land/simplified_geometries_slow_…

…queries

Simplified geometries slow queries

application/data_access/entity_queries.py

@@ -75,20 +75,51 @@ def get_entity_search(session: Session, parameters: dict):
     count_query = session.query(func.count()).select_from(subquery)
     count = count_query.scalar()
 
-    # get entities
     query_args = [EntityOrm]
     query = session.query(*query_args)
     query = _apply_base_filters(query, params)
     query = _apply_date_filters(query, params)
     query = _apply_location_filters(session, query, params)
     query = _apply_period_option_filter(query, params)
     query = _apply_limit_and_pagination_filters(query, params)
-
+    query = _apply_exclusion_filters(
+        query, params
+    )  # Build the query without excluded params
     entities = query.all()
     entities = [entity_factory(entity_orm) for entity_orm in entities]
     return {"params": params, "count": count, "entities": entities}
 
 
+def _apply_exclusion_filters(query, params):
+    exclude_fields = params.get("exclude_field", [])
+    if exclude_fields:
+        # Split the comma-separated string into a list of individual fields
+        split_strings = [
+            s.strip() for sub in exclude_fields for s in sub.split(",") if s
+        ]
+        exclude_fields = set(split_strings)
+        all_columns = {column.name for column in EntityOrm.__table__.columns}
+
+        # Exclude fields not present in the table schema
+        exclude_fields = {field for field in exclude_fields if field in all_columns}
+        # Dynamically construct the selected columns by excluding the specified fields
+        selected_columns = [
+            column
+            for column in EntityOrm.__table__.columns
+            if column.name not in exclude_fields
+        ]
+
+        if not selected_columns:
+            raise ValueError(
+                "No columns left to select after exclusions. Please check the field names."
+            )
+
+        # Modify the query to select only the desired columns
+        query = query.with_entities(*selected_columns)
+
+    return query
+
+
 def lookup_entity_link(
     session: Session, reference: str, dataset: str, organisation_entity: int
 ):
@@ -181,7 +212,8 @@ def _apply_location_filters(session, query, params):
                     EntityOrm.geometry.is_not(None),
                     func.ST_IsValid(EntityOrm.geometry),
                     spatial_function(
-                        EntityOrm.geometry, func.ST_GeomFromText(geometry, 4326)
+                        EntityOrm.geometry,
+                        func.ST_GeomFromText(geometry, 4326),
                     ),
                 ),
                 and_(
@@ -195,7 +227,6 @@ def _apply_location_filters(session, query, params):
         )
     if clauses:
         query = query.filter(or_(*clauses))
-
     intersecting_entities = params.get("geometry_entity", [])
     if intersecting_entities:
         intersecting_entities_query = (

tests/integration/test_entity_search.py

@@ -538,4 +538,23 @@ def test_search_entity_by_organisation_curie(test_data, params, db_session):
     assert entity.reference == expected_entity["reference"]
 
 
+def test_search_entity_with_exclude_parameter(test_data, params, db_session):
+    from tests.test_data.wkt_data import intersects_with_brownfield_entity as brownfield
+    from tests.test_data.wkt_data import intersects_with_greenspace_entity as greenspace
+
+    params["geometry"] = [brownfield, greenspace]
+    params["geometry_relation"] = GeometryRelation.intersects.name
+    params["exclude_field"] = [""]
+    result = get_entity_search(db_session, params)
+    for entity in result["entities"]:
+        assert entity.geometry is not None
+    assert result["count"] == 2
+
+    params["exclude_field"] = ["geometry"]
+    result = get_entity_search(db_session, params)
+    for entity in result["entities"]:
+        assert entity.geometry is None
+    assert result["count"] == 2
+
+
 # TODO test cases for contains, within