Add spatial unit classes

CHANGELOG.md

@@ -9,13 +9,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 
 - The dev provisioning Ansible playbook now automatically generates an SSH key pair for the `flowkit` user. [#892](https://github.com/Flowminder/FlowKit/issues/892)
+- Added new classes to represent spatial units in FlowMachine.
+- Added a `Geography` query class, to get geography data for a spatial unit.
 - FlowAPI's 'joined_spatial_aggregate' endpoint now exposes unique location counts.[#949](https://github.com/Flowminder/FlowKit/issues/949)
 - FlowAPI's 'joined_spatial_aggregate' endpoint now exposes subscriber degree.[#969](https://github.com/Flowminder/FlowKit/issues/969)
 - Flowdb now contains an auxiliary table to record outcomes of queries that can be run as part of the regular ETL process [#988](https://github.com/Flowminder/FlowKit/issues/988)
 
 ### Changed
 
 - The quick-start script now only pulls the docker images for the services that are actually started up. [#898](https://github.com/Flowminder/FlowKit/issues/898)
+- Location-related FlowMachine queries now take a `spatial_unit` parameter instead of `level`.
 - The quick-start script now uses the environment variable `GIT_REVISION` to control the version to be deployed.
 - Create token page permission and spatial aggregation checkboxes are now hidden by default.[#834](https://github.com/Flowminder/FlowKit/issues/834)
 - The flowetl mounted directories `archive, dump, ingest, quarantine` were replaced with a single `files` directory and files are no longer moved. [#946](https://github.com/Flowminder/FlowKit/issues/946)
@@ -28,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - The quick-start script now works correctly with branches. [#902](https://github.com/Flowminder/FlowKit/issues/902)
 
 ### Removed
+- Removed `cell_mappings.py`, `get_columns_for_level` and `BadLevelError`.
 
 - The FlowDB tables `infrastructure.countries` and `infrastructure.operators` have been removed. [#958](https://github.com/Flowminder/FlowKit/issues/958)
 

docs/cache_queries.py

@@ -16,33 +16,41 @@
 
 print("Constructing query objects")
 
+admin1_spatial_unit = flowmachine.core.make_spatial_unit("admin", level=1)
+admin3_spatial_unit = flowmachine.core.make_spatial_unit("admin", level=3)
+vsite_spatial_unit = flowmachine.core.make_spatial_unit("versioned-site")
+vcell_spatial_unit = flowmachine.core.make_spatial_unit("versioned-cell")
+
 # FlowClient example usage
 example_usage_queries = [
     flowmachine.features.utilities.spatial_aggregates.SpatialAggregate(
         locations=flowmachine.features.daily_location(
-            date="2016-01-01", level="admin3", method="last"
+            date="2016-01-01", spatial_unit=admin3_spatial_unit, method="last"
         )
     ),
     flowmachine.features.utilities.spatial_aggregates.SpatialAggregate(
         locations=flowmachine.features.ModalLocation(
             *[
                 flowmachine.features.daily_location(
-                    date=dl_date, level="admin3", method="last"
+                    date=dl_date, spatial_unit=admin3_spatial_unit, method="last"
                 )
                 for dl_date in pd.date_range("2016-01-01", "2016-01-03", freq="D")
             ]
         )
     ),
     flowmachine.features.Flows(
         flowmachine.features.daily_location(
-            date="2016-01-01", level="admin1", method="last"
+            date="2016-01-01", spatial_unit=admin1_spatial_unit, method="last"
         ),
         flowmachine.features.daily_location(
-            date="2016-01-07", level="admin1", method="last"
+            date="2016-01-07", spatial_unit=admin1_spatial_unit, method="last"
         ),
     ),
     flowmachine.features.TotalLocationEvents(
-        start="2016-01-01", stop="2016-01-08", level="admin3", interval="hour"
+        start="2016-01-01",
+        stop="2016-01-08",
+        spatial_unit=admin3_spatial_unit,
+        interval="hour",
     ),
 ]
 
@@ -57,7 +65,9 @@
         locations=flowmachine.features.ModalLocation(
             *[
                 flowmachine.features.daily_location(
-                    date=dl_date.strftime("%Y-%m-%d"), level="admin3", method="last"
+                    date=dl_date.strftime("%Y-%m-%d"),
+                    spatial_unit=admin3_spatial_unit,
+                    method="last",
                 )
                 for dl_date in dates
             ]
@@ -69,15 +79,19 @@
         flowmachine.features.ModalLocation(
             *[
                 flowmachine.features.daily_location(
-                    date=dl_date.strftime("%Y-%m-%d"), level="admin3", method="last"
+                    date=dl_date.strftime("%Y-%m-%d"),
+                    spatial_unit=admin3_spatial_unit,
+                    method="last",
                 )
                 for dl_date in date_ranges["benchmark"]
             ]
         ),
         flowmachine.features.ModalLocation(
             *[
                 flowmachine.features.daily_location(
-                    date=dl_date.strftime("%Y-%m-%d"), level="admin3", method="last"
+                    date=dl_date.strftime("%Y-%m-%d"),
+                    spatial_unit=admin3_spatial_unit,
+                    method="last",
                 )
                 for dl_date in date_ranges[period2]
             ]
@@ -134,8 +148,10 @@
         meaningful_locations=flowmachine.features.MeaningfulLocations(
             clusters=flowmachine.features.HartiganCluster(
                 calldays=flowmachine.features.CallDays(
-                    subscriber_locations=flowmachine.features.subscriber_locations(
-                        start="2016-01-01", stop="2016-01-07", level="versioned-site"
+                    subscriber_locations=flowmachine.features.SubscriberLocations(
+                        start="2016-01-01",
+                        stop="2016-01-07",
+                        spatial_unit=vsite_spatial_unit,
                     )
                 ),
                 radius=1.0,
@@ -148,20 +164,22 @@
                 stop="2016-01-07",
                 score_hour=hour_scores,
                 score_dow=day_scores,
-                level="versioned-site",
+                spatial_unit=vsite_spatial_unit,
             ),
             label=label,
         ),
-        level="admin3",
+        spatial_unit=admin3_spatial_unit,
     )
     for label in ["home", "work"]
 ] + [
     flowmachine.features.MeaningfulLocationsOD(
         meaningful_locations_a=flowmachine.features.MeaningfulLocations(
             clusters=flowmachine.features.HartiganCluster(
                 calldays=flowmachine.features.CallDays(
-                    subscriber_locations=flowmachine.features.subscriber_locations(
-                        start="2016-01-01", stop="2016-01-07", level="versioned-site"
+                    subscriber_locations=flowmachine.features.SubscriberLocations(
+                        start="2016-01-01",
+                        stop="2016-01-07",
+                        spatial_unit=vsite_spatial_unit,
                     )
                 ),
                 radius=1.0,
@@ -174,15 +192,17 @@
                 stop="2016-01-07",
                 score_hour=hour_scores,
                 score_dow=day_scores,
-                level="versioned-site",
+                spatial_unit=vsite_spatial_unit,
             ),
             label="home",
         ),
         meaningful_locations_b=flowmachine.features.MeaningfulLocations(
             clusters=flowmachine.features.HartiganCluster(
                 calldays=flowmachine.features.CallDays(
-                    subscriber_locations=flowmachine.features.subscriber_locations(
-                        start="2016-01-01", stop="2016-01-07", level="versioned-site"
+                    subscriber_locations=flowmachine.features.SubscriberLocations(
+                        start="2016-01-01",
+                        stop="2016-01-07",
+                        spatial_unit=vsite_spatial_unit,
                     )
                 ),
                 radius=1.0,
@@ -195,11 +215,11 @@
                 stop="2016-01-07",
                 score_hour=hour_scores,
                 score_dow=day_scores,
-                level="versioned-site",
+                spatial_unit=vsite_spatial_unit,
             ),
             label="work",
         ),
-        level="admin3",
+        spatial_unit=admin3_spatial_unit,
     )
 ]
 
@@ -209,22 +229,25 @@
         start="2016-01-01",
         stop="2016-01-07",
         table="events.mds",
-        level="versioned-cell",
+        spatial_unit=vcell_spatial_unit,
         interval="hour",
     )
 ]
 
 # Cell Towers Per Region
 cell_towers_per_region_queries = [
     flowmachine.features.TotalNetworkObjects(
-        start="2016-01-01", stop="2016-01-08", level="admin3", total_by="month"
+        start="2016-01-01",
+        stop="2016-01-08",
+        spatial_unit=admin3_spatial_unit,
+        total_by="month",
     )
 ]
 
 # Unique Subscriber Counts
 unique_subscriber_counts_queries = [
     flowmachine.features.UniqueSubscriberCounts(
-        start="2016-01-01", stop="2016-01-08", level="admin3"
+        start="2016-01-01", stop="2016-01-08", spatial_unit=admin3_spatial_unit
     )
 ]
 

docs/source/worked_examples/cell-towers-per-region.ipynb

@@ -157,14 +157,13 @@
    "source": [
     "towers_per_admin3_geodataframe = (\n",
     "    regions_geodataframe.join(\n",
-    "        towers_per_admin3.set_index(\"pcod\"), on=\"admin3pcod\", how=\"left\"\n",
+    "        towers_per_admin3.set_index(\"pcod\"), on=\"pcod\", how=\"left\"\n",
     "    )\n",
     "    .fillna(0)\n",
     "    .drop(columns=[\"centroid\", \"datetime\"])\n",
     "    .rename(\n",
     "        columns={\n",
-    "            \"admin3pcod\": \"P-code\",\n",
-    "            \"admin3name\": \"Name\",\n",
+    "            \"pcod\": \"P-code\",\n",
     "            \"value\": \"Number of towers\",\n",
     "        }\n",
     "    )\n",

docs/source/worked_examples/commuting-patterns.ipynb

@@ -199,12 +199,12 @@
     "    regions_geodataframe.drop(columns=\"centroid\")\n",
     "    .join(\n",
     "        home_locations.drop(columns=\"label\").set_index(\"pcod\"),\n",
-    "        on=\"admin3pcod\",\n",
+    "        on=\"pcod\",\n",
     "        how=\"left\",\n",
     "    )\n",
     "    .join(\n",
     "        work_locations.drop(columns=\"label\").set_index(\"pcod\"),\n",
-    "        on=\"admin3pcod\",\n",
+    "        on=\"pcod\",\n",
     "        lsuffix=\"_home\",\n",
     "        rsuffix=\"_work\",\n",
     "        how=\"left\",\n",
@@ -215,8 +215,7 @@
     "# Rename columns for map labels\n",
     "locations_geodataframe = locations_geodataframe.rename(\n",
     "    columns={\n",
-    "        \"admin3pcod\": \"P-code\",\n",
-    "        \"admin3name\": \"Name\",\n",
+    "        \"pcod\": \"P-code\",\n",
     "        \"total_home\": \"Total (home)\",\n",
     "        \"total_work\": \"Total (work)\",\n",
     "    }\n",
@@ -341,10 +340,10 @@
     "# Join location counts to geography data\n",
     "commuters_geodataframe = (\n",
     "    regions_geodataframe.drop(columns=\"centroid\")\n",
-    "    .join(commuters_into_region, on=\"admin3pcod\", how=\"left\")\n",
+    "    .join(commuters_into_region, on=\"pcod\", how=\"left\")\n",
     "    .join(\n",
     "        commuters_out_from_region,\n",
-    "        on=\"admin3pcod\",\n",
+    "        on=\"pcod\",\n",
     "        lsuffix=\"_in\",\n",
     "        rsuffix=\"_out\",\n",
     "        how=\"left\",\n",
@@ -355,8 +354,7 @@
     "# Rename columns for map labels\n",
     "commuters_geodataframe = commuters_geodataframe.rename(\n",
     "    columns={\n",
-    "        \"admin3pcod\": \"P-code\",\n",
-    "        \"admin3name\": \"Name\",\n",
+    "        \"pcod\": \"P-code\",\n",
     "        \"total_in\": \"Commuters in\",\n",
     "        \"total_out\": \"Commuters out\",\n",
     "    }\n",

docs/source/worked_examples/flows-above-normal.ipynb

@@ -253,12 +253,12 @@
     "        home_locations_results[period]\n",
     "        .set_index(\"pcod\")\n",
     "        .rename(columns={\"total\": f\"Total ({period} period)\"}),\n",
-    "        on=\"admin3pcod\",\n",
+    "        on=\"pcod\",\n",
     "        how=\"left\",\n",
     "    ).fillna(0)\n",
     "\n",
     "home_locations_geodataframe = home_locations_geodataframe.rename(\n",
-    "    columns={\"admin3pcod\": \"P-code\", \"admin3name\": \"Name\"}\n",
+    "    columns={\"pcod\": \"P-code\"}\n",
     ")"
    ]
   },
@@ -389,7 +389,7 @@
    "outputs": [],
    "source": [
     "in_out_flows_geodataframe = (\n",
-    "    regions_geodataframe.set_index(\"admin3pcod\")\n",
+    "    regions_geodataframe.set_index(\"pcod\")\n",
     "    .join(\n",
     "        [\n",
     "            inflows_above_normal.rename(\n",
@@ -407,7 +407,7 @@
     "\n",
     "in_out_flows_geodataframe = in_out_flows_geodataframe.drop(\n",
     "    columns=\"centroid\"\n",
-    ").rename(columns={\"admin3pcod\": \"P-code\", \"admin3name\": \"Name\"})"
+    ").rename(columns={\"pcod\": \"P-code\"})"
    ]
   },
   {

docs/source/worked_examples/mobile-data-usage.ipynb

@@ -28,6 +28,7 @@
    "outputs": [],
    "source": [
     "import flowmachine\n",
+    "from flowmachine.core import make_spatial_unit\n",
     "import os\n",
     "import numpy as np\n",
     "import geopandas as gpd\n",
@@ -82,7 +83,7 @@
     "    start=\"2016-01-01\",\n",
     "    stop=\"2016-01-07\",\n",
     "    table=\"events.mds\",\n",
-    "    level=\"versioned-cell\",\n",
+    "    spatial_unit=make_spatial_unit(\"versioned-cell\"),\n",
     "    interval=\"hour\",\n",
     ")"
    ]

docs/source/worked_examples/unique-subscriber-counts.ipynb

@@ -156,14 +156,13 @@
    "source": [
     "subscribers_per_admin3_geodataframe = (\n",
     "    regions_geodataframe.join(\n",
-    "        subscribers_per_admin3.set_index(\"pcod\"), on=\"admin3pcod\", how=\"left\"\n",
+    "        subscribers_per_admin3.set_index(\"pcod\"), on=\"pcod\", how=\"left\"\n",
     "    )\n",
     "    .fillna(0)\n",
     "    .drop(columns=[\"centroid\"])\n",
     "    .rename(\n",
     "        columns={\n",
-    "            \"admin3pcod\": \"P-code\",\n",
-    "            \"admin3name\": \"Name\",\n",
+    "            \"pcod\": \"P-code\",\n",
     "            \"unique_subscriber_counts\": \"Number of subscribers\",\n",
     "        }\n",
     "    )\n",

flowapi/flowapi/geography.py

@@ -54,7 +54,11 @@ async def get_geography(aggregation_unit):
     )
 
     if reply["status"] == "error":
-        return jsonify({"status": "Error", "msg": "Internal server error"}), 500
+        try:
+            msg = reply["msg"]
+        except KeyError:
+            msg = "Internal server error"
+        return jsonify({"status": "Error", "msg": msg}), 500
 
     try:
         query_state = reply["payload"]["query_state"]

flowmachine/flowmachine/core/__init__.py

@@ -13,8 +13,10 @@
 from .geotable import GeoTable
 from .init import connect
 from .logging import init_logging, set_log_level
-from .join_to_location import JoinToLocation
+from .spatial_unit import make_spatial_unit
+from .join_to_location import JoinToLocation, location_joined_query
 from .custom_query import CustomQuery
+from .grid import Grid
 
 sub_modules = ["errors", "mixins", "api"]
 
@@ -24,8 +26,11 @@
     "GeoTable",
     "Connection",
     "connect",
+    "make_spatial_unit",
     "JoinToLocation",
+    "location_joined_query",
     "CustomQuery",
+    "Grid",
 ]
 
 __all__ = methods + sub_modules

flowmachine/flowmachine/core/errors/__init__.py

@@ -9,8 +9,13 @@
 from .flowmachine_errors import (
     NameTooLongError,
     NotConnectedError,
-    BadLevelError,
+    InvalidSpatialUnitError,
     MissingDateError,
 )
 
-__all__ = ["NameTooLongError", "NotConnectedError", "BadLevelError", "MissingDateError"]
+__all__ = [
+    "NameTooLongError",
+    "NotConnectedError",
+    "InvalidSpatialUnitError",
+    "MissingDateError",
+]